[ofw] [PATCH] ipoib-CM 7 of 13
Smith, Stan
stan.smith at intel.com
Tue Jan 11 18:31:29 PST 2011
signed-off-by: stan smith <stan.smith at intel.com>
--- A/ulp/ipoib_NDIS6_CM/kernel/ipoib_endpoint.h Tue Jan 11 17:56:40 2011
+++ B/ulp/ipoib_NDIS6_CM/kernel/ipoib_endpoint.h Tue Jan 11 16:46:43 2011
@@ -46,37 +46,74 @@
#include "ipoib_debug.h"
-typedef struct _endpt_buf_mgr
+typedef struct _cm_buf_mgr
{
cl_qpool_t recv_pool;
- NDIS_HANDLE h_packet_pool;
- NDIS_HANDLE h_buffer_pool;
- cl_qlist_t posted_list;
+ NDIS_HANDLE h_nbl_pool;
+ cl_spinlock_t lock;
+ cl_qlist_t oop_list;
+ long posted;
+ int32_t recv_pool_depth;
boolean_t pool_init;
-} endpt_buf_mgr_t;
+} cm_buf_mgr_t;
+/*
+* FIELDS
+* recv_pool
+* recv descriptor pool - shared by all endpoints & posted to SRQ.
+*
+* h_nbl_pool
+* handle to the pool of NDIS NETWORK_BUFFER_LISTs
+*
+* lock
+* serialize access to the recv_pool & oop_list.
+*
+* oop_list
+* list of recv pool buffers which are Out-Of-Pool.
+* (SRQ bound QPs do not flush buffers on transition to QP Error state, this
+* list tracks recv pool buffers by hand).
+*
+* posted
+* Number of outstanding recv pool elements on the oop_list.
+* Normally these pool elements are posted to the SRQ (for all endpoints)
+* CM recv code wants to keep (posted == params.rq_depth).
+*
+* recv_pool_depth
+* Total number of recv descriptor pool elements.
+*
+* pool_init
+* boolean: TRUE == recv pool has been initialized.
+*/
typedef struct _endpt_recv_mgr
{
- int32_t depth;
- int32_t rq_depth;
- //NDIS60
- //NDIS_PACKET **recv_NBL_array;
- NET_BUFFER_LIST *recv_lst_array;
+ NET_BUFFER_LIST *NBL;
+ cl_qlist_t done_list;
} endpt_recv_mgr_t;
+/*
+* FIELDS
+* NBL
+* Linked list of one or more NBL's (Network Buffer Lists) chained together.
+* Each NBL points @ a ipoib_cm_desc_t->p_buf of RC received packet data.
+*
+* done_list
+* list of completed WR (work Requests) which have passed recv filtering.
+* Passed in that the WRs can be converted into NBLs and passed up to NDIS.
+*/
typedef enum _cm_state
{
IPOIB_CM_DISCONNECTED,
- IPOIB_CM_INIT,
- IPOIB_CM_CONNECT,
+ IPOIB_CM_QUEUED_TO_CONNECT,
+ IPOIB_CM_CONNECTING,
IPOIB_CM_CONNECTED,
IPOIB_CM_LISTEN,
IPOIB_CM_DREP_SENT,
IPOIB_CM_DREQ_SENT,
- IPOIB_CM_REJ_RECVD,
+ IPOIB_CM_DISCONNECT_CLEANUP,
IPOIB_CM_DESTROY
+
} cm_state_t;
typedef struct _cm_private_data
@@ -90,15 +127,59 @@
ib_net64_t service_id;
cm_private_data_t private_data;
ib_qp_handle_t h_send_qp;
- ib_qp_handle_t h_recv_qp;
- ib_qp_handle_t h_work_qp;
+ ib_qp_handle_t h_send_qp_err;
ib_cq_handle_t h_send_cq;
+ ib_qp_handle_t h_recv_qp;
ib_cq_handle_t h_recv_cq;
ib_listen_handle_t h_cm_listen;
cm_state_t state;
} endpt_conn_t;
+/*
+* FIELDS
+* service_id
+* listen() on this service ID
+*
+* private_data
+* private data received from remote side.
+*
+* h_send_qp
+* RC qp for send
+*
+* h_send_qp_err
+* If !null, then copy of h_send_qp prior to setting h_send_qp == NULL.
+* QP is in error state, awaiting destroy.
+*
+* h_recv_qp
+* Rx RC qp handle
+*
+* h_send_cq
+* Tx CQ handle
+*
+* h_recv_cq
+* Rx CQ handle
+*
+* h_cm_listen
+* listen()ing CM handle.
+*
+* state
+* connection state for the active connection only.
+*
+* NOTES
+* IpoIB Connect Mode (CM) connection protocol in a nutshell.
+* An IPoIB interface encodes it CM capability in the hardware address it publishes.
+* Once NDIS hands a Network Buffer to IPoIB for transmission (unicast), IPoIB needs
+* to resolve the hardware address to IB adress information (LID, SL etc.).
+* Once resolution is complete, an RC connection is forged with the remote host
+* using IBAL CM.
+* The connection process is symetrical - in that, for two communicating hosts
+* there are two RC connections such that each connection works half duplex.
+* CM state variable is that of the 'active' connection (h_send_qp), the passive
+* side (h_recv_qp) setup does not alter the CM state variable.
+*
+*********/
+
typedef struct _ipoib_endpt
{
cl_obj_t obj;
@@ -111,30 +192,44 @@
ib_query_handle_t h_query;
ib_mcast_handle_t h_mcast;
mac_addr_t mac;
- ib_gid_t dgid;
net16_t dlid;
+ ib_gid_t dgid;
net32_t qpn;
- uint8_t cm_flag;
ib_av_handle_t h_av;
- endpt_conn_t conn;
-
ib_al_ifc_t *p_ifc;
boolean_t is_in_use;
boolean_t is_mcast_listener;
+ endpt_recv_mgr_t cm_recv;
+ endpt_conn_t conn;
+ uint32_t tx_mtu;
+ uint8_t cm_flag;
+ uint8_t cm_rx_flushing;
+ uint8_t cm_ep_destroy;
+ char tag[24]; // <(Broad/Multi)-cast or 0xLID> string
+
} ipoib_endpt_t;
/*
* FIELDS
-* mac_item
-* Map item for storing the endpoint in a map. The key is the
-* destination MAC address.
+* obj
+* Pointer to the EndPoint object proper.
*
-* lid_item
-* Map item for storing the endpoint in a map. The key is the
-* destination LID.
+* rel
+* Object relations - used to convert from EndPoint to Port struct pointer.
*
* gid_item
-* Map item for storing the endpoint in a map. The key is the
-* destination GID.
+* Map item for storing the endpoint in a map. key is destination GID.
+*
+* lid_item
+* Map item for storing the endpoint in a map. key is destination LID.
+*
+* conn_item
+* Map item for storing the endpoint in a connect map. key is hardware MAC.
+*
+* mac_item
+* Map item for storing the endpoint in a map. key is destination MAC address.
+*
+* list_item
+* used when emdpoint is on the connection list.
*
* h_query
* Query handle for cancelling SA queries.
@@ -143,31 +238,52 @@
* For multicast endpoints, the multicast handle.
*
* mac
-* MAC address.
-*
-* dgid
-* Destination GID.
+* MAC address; next 2 bytes make for even alignment.
*
* dlid
* Destination LID. The destination LID is only set for endpoints
* that are on the same subnet. It is used as key in the LID map.
*
+* dgid
+* Destination GID.
+*
* qpn
-* Destination queue pair number.
+* Destination UD queue pair number.
*
* h_av
* Address vector for sending data.
*
-* expired
-* Flag to indicate that the endpoint should be flushed.
-*
-* connection
-* for connected mode endpoints
-*
* p_ifc
* Reference to transport functions, can be used
* while endpoint is not attached to port yet.
*
+* is_in_use
+* Endpoint is a member of a mcast group.
+*
+* is_mcast_listener
+*
+* cm_recv
+* Manage NDIS NBLs (Network Buffer List) and completed recv work-requests.
+*
+* conn
+* for connected mode endpoints, IB RC connection info; includes conn state.
+*
+* tx_mtu
+* current MTU; starts as UD MTU, when CM connected then
+* tx_mtu = params.cm_payload_mtu, otherwise revert back to UD mtu.
+*
+* cm_flag
+* != 0 implies CM capable.
+*
+* cm_rx_flushing
+* != 0, CM is flushing SRQ QPs; defer object destroy until done flushing.
+*
+* cm_ep_destroy
+* SRQ async error routine should destroy the EP object.
+*
+* tag
+* Endpoint tag string: asciz string 'lid 0xNNN'
+*
* NOTES
* If the h_mcast member is set, the endpoint is never expired.
*********/
@@ -198,7 +314,7 @@
cl_obj_ref( &p_endpt->obj );
#if DBG
IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,
- ("[%#x] Endpt refcount raised to %d\n", p_endpt, p_endpt->obj.ref_cnt));
+ ("Endpt %s ++refcount %d\n", p_endpt->tag, p_endpt->obj.ref_cnt));
#endif
/*
* Anytime we reference the endpoint, we're either receiving data
@@ -216,10 +332,12 @@
cl_obj_deref( &p_endpt->obj );
#if DBG
IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_OBJ,
- ("[%#x] Endpt refcount decremented to %d\n", p_endpt, p_endpt->obj.ref_cnt));
+ ("Endpt %s --refcount %d\n", p_endpt->tag, p_endpt->obj.ref_cnt));
#endif
}
+void ipoib_endpt_cm_mgr_thread(
+ IN void* p_context );
NDIS_STATUS
ipoib_endpt_queue(
@@ -230,17 +348,21 @@
ipoib_endpt_parent(
IN ipoib_endpt_t* const p_endpt );
-inline cm_state_t
+void
+endpt_unmap_conn_dgid(
+ IN ipoib_port_t* const p_port,
+ IN ipoib_endpt_t* const p_endpt );
+
+static inline cm_state_t
endpt_cm_set_state(
IN ipoib_endpt_t* const p_endpt,
IN cm_state_t state )
{
- return (cm_state_t)InterlockedExchange(
- (volatile LONG *)&p_endpt->conn.state,
- (LONG)state );
+ return (cm_state_t) InterlockedExchange( (volatile LONG *)&p_endpt->conn.state,
+ (LONG)state );
}
-inline cm_state_t
+static inline cm_state_t
endpt_cm_get_state(
IN ipoib_endpt_t* const p_endpt )
{
@@ -250,19 +372,42 @@
}
ib_api_status_t
-endpt_cm_create_qp(
- IN ipoib_endpt_t* const p_endpt,
- IN ib_qp_handle_t* const p_h_qp );
+endpt_cm_connect(
+ IN ipoib_endpt_t* const p_endpt );
-ib_api_status_t
-ipoib_endpt_connect(
+void
+endpt_queue_cm_connection(
+ IN ipoib_port_t* const p_port,
IN ipoib_endpt_t* const p_endpt );
-int32_t
-endpt_cm_recv_mgr_filter(
+void
+cm_release_resources(
+ IN ipoib_port_t* const p_port,
IN ipoib_endpt_t* const p_endpt,
- IN ib_wc_t* const p_done_wc_list,
- OUT cl_qlist_t* const p_done_list,
- OUT cl_qlist_t* const p_bad_list );
+ IN int which_res );
+
+void
+cm_destroy_recv_resources(
+ IN ipoib_port_t* const p_port,
+ IN ipoib_endpt_t* const p_endpt );
+
+
+char *
+cm_get_state_str(
+ IN cm_state_t );
+
+char *
+get_eth_packet_type_str(
+ IN net16_t);
+
+char *
+get_IP_protocol_str(
+ IN uint8_t);
+
+
+#if DBG
+
+void decode_enet_pkt( char *preFix, void *hdr, int len, char *postFix );
+#endif
#endif /* _IPOIB_ENDPOINT_H_ */
-------------- next part --------------
A non-text attachment was scrubbed...
Name: ipoib_endpoint.h.patch
Type: application/octet-stream
Size: 9718 bytes
Desc: ipoib_endpoint.h.patch
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20110111/19e7b776/attachment.obj>
More information about the ofw
mailing list