[ofw][patch] mcast garbage collector
Slava Strebkov
slavas at voltaire.com
Mon Jul 14 03:23:06 PDT 2008
This is a garbage collector implementation which is using custom DPC
callback.
Garbage collector is always enabled, allowing sending IB leave to groups
opened by multicast sender.
Index: ulp/ipoib/kernel/ipoib_adapter.h
===================================================================
--- ulp/ipoib/kernel/ipoib_adapter.h (revision 1394)
+++ ulp/ipoib/kernel/ipoib_adapter.h (working copy)
@@ -75,7 +75,7 @@
uint32_t payload_mtu;
uint32_t xfer_block_size;
mac_addr_t conf_mac;
-
+ uint32_t mc_leave_rescan;
} ipoib_params_t;
/*
* FIELDS
Index: ulp/ipoib/kernel/ipoib_driver.c
===================================================================
--- ulp/ipoib/kernel/ipoib_driver.c (revision 1394)
+++ ulp/ipoib/kernel/ipoib_driver.c (working copy)
@@ -153,7 +153,8 @@
{NDIS_STRING_CONST("SaTimeout"), 1,
IPOIB_OFFSET(sa_timeout), IPOIB_SIZE(sa_timeout),
1000, 250, UINT_MAX},
{NDIS_STRING_CONST("SaRetries"), 1,
IPOIB_OFFSET(sa_retry_cnt), IPOIB_SIZE(sa_retry_cnt), 10,
1, UINT_MAX},
{NDIS_STRING_CONST("RecvRatio"), 1,
IPOIB_OFFSET(recv_pool_ratio), IPOIB_SIZE(recv_pool_ratio), 1,
1, 10},
- {NDIS_STRING_CONST("PayloadMtu"), 1,
IPOIB_OFFSET(payload_mtu), IPOIB_SIZE(payload_mtu),
2044, 60, 4092}
+ {NDIS_STRING_CONST("PayloadMtu"), 1,
IPOIB_OFFSET(payload_mtu), IPOIB_SIZE(payload_mtu),
2044, 60, 4092},
+ {NDIS_STRING_CONST("MCLeaveRescan"), 1,
IPOIB_OFFSET(mc_leave_rescan), IPOIB_SIZE(mc_leave_rescan),
260, 1, 3600}
};
#define IPOIB_NUM_REG_PARAMS (sizeof (HCARegTable) /
sizeof(IPOIB_REG_ENTRY))
Index: ulp/ipoib/kernel/ipoib_endpoint.h
===================================================================
--- ulp/ipoib/kernel/ipoib_endpoint.h (revision 1394)
+++ ulp/ipoib/kernel/ipoib_endpoint.h (working copy)
@@ -62,7 +62,8 @@
ib_av_handle_t h_av;
boolean_t expired;
ib_al_ifc_t *p_ifc;
-
+ boolean_t is_in_use;
+ boolean_t is_mcast_listener;
} ipoib_endpt_t;
/*
* FIELDS
Index: ulp/ipoib/kernel/ipoib_port.c
===================================================================
--- ulp/ipoib/kernel/ipoib_port.c (revision 1394)
+++ ulp/ipoib/kernel/ipoib_port.c (working copy)
@@ -67,7 +67,8 @@
ipoib_port_t *gp_ipoib_port;
#endif
-
+static void __port_mcast_garbage_dpc(KDPC *p_gc_dpc,void *context,void
*s_arg1, void *s_arg2);
+static void __port_do_mcast_garbage(ipoib_port_t* const p_port
);
/***********************************************************************
*******
*
* Declarations
@@ -653,6 +654,9 @@
p_adapter->p_ifc->get_err_str( status )) );
return status;
}
+ /* Initialize multicast garbage collector timer and DPC object */
+
KeInitializeDpc(&p_port->gc_dpc,(PKDEFERRED_ROUTINE)__port_mcast_garbage
_dpc,p_port);
+ KeInitializeTimerEx(&p_port->gc_timer,SynchronizationTimer);
/* We only ever destroy from the PnP callback thread. */
cl_status = cl_obj_init( &p_port->obj, CL_DESTROY_SYNC,
@@ -748,6 +752,7 @@
p_port = PARENT_STRUCT( p_obj, ipoib_port_t, obj );
+ KeCancelTimer(&p_port->gc_timer);
__endpt_mgr_destroy( p_port );
__recv_mgr_destroy( p_port );
__send_mgr_destroy( p_port );
@@ -2132,6 +2137,8 @@
p_eth->hdr.src = p_src->mac;
p_eth->hdr.dst = p_dst->mac;
+ CL_ASSERT(p_dst->h_mcast != NULL);
+ p_dst->is_in_use = TRUE;
IPOIB_EXIT( IPOIB_DBG_RECV );
return IB_SUCCESS;
}
@@ -3130,6 +3137,9 @@
return status;
}
+ p_endpt->is_mcast_listener = TRUE;
+ p_endpt->is_mcast_listener = FALSE;
+ __port_do_mcast_garbage(p_port);
static NDIS_STATUS
__send_mgr_filter_udp(
@@ -3601,6 +3611,9 @@
("Multicast Mac - trying to join.\n") );
return NDIS_STATUS_PENDING;
}
+ CL_ASSERT( (*pp_endpt) );
+ CL_ASSERT((*pp_endpt)->h_mcast != NULL);
+ (*pp_endpt)->is_in_use = TRUE;
}
IPOIB_EXIT( IPOIB_DBG_SEND );
@@ -4731,6 +4744,7 @@
ib_query_req_t query;
ib_user_query_t info;
ib_portinfo_record_t port_rec;
+ LARGE_INTEGER gc_due_time;
IPOIB_ENTER( IPOIB_DBG_INIT );
@@ -4778,6 +4792,10 @@
return;
}
+ /* garbage collector timer is needed when link is active */
+ gc_due_time.QuadPart =
-(int64_t)(((uint64_t)p_port->p_adapter->params.mc_leave_rescan *
2000000) * 10);
+ KeSetTimerEx(&p_port->gc_timer,gc_due_time,
+
(LONG)p_port->p_adapter->params.mc_leave_rescan*1000,&p_port->gc_dpc);
IPOIB_EXIT( IPOIB_DBG_INIT );
}
@@ -5253,6 +5271,8 @@
p_port->p_adapter->hung = TRUE;
return;
}
+ /* garbage collector timer is not needed when link is down */
+ KeCancelTimer(&p_port->gc_timer);
KeResetEvent(&p_port->leave_mcast_event);
@@ -5681,6 +5701,8 @@
&p_port->endpt_mgr.lid_endpts, p_endpt->dlid,
&p_endpt->lid_item );
CL_ASSERT( p_qitem == &p_endpt->lid_item );
}
+ /* set flag that endpoint is use */
+ p_endpt->is_in_use = TRUE;
cl_obj_unlock( &p_port->obj );
/* Try to send all pending sends. */
@@ -5737,6 +5759,81 @@
IPOIB_EXIT( IPOIB_DBG_MCAST );
}
+static void __port_do_mcast_garbage(ipoib_port_t* const p_port)
+{
+ const mac_addr_t DEFAULT_MCAST_GROUP = {0x01, 0x00, 0x5e, 0x00,
0x00, 0x01};
+ /* Do garbage collecting... */
+ cl_map_item_t *p_item;
+ ipoib_endpt_t *p_endpt;
+ cl_qlist_t destroy_mc_list;
+ uint8_t cnt;
+ const static GC_MAX_LEAVE_NUM = 80;
+ cl_qlist_init( &destroy_mc_list );
+ cl_obj_lock( &p_port->obj );
+ cnt = 0;
+ p_item = cl_qmap_head( &p_port->endpt_mgr.mac_endpts );
+ while( (p_item != cl_qmap_end( &p_port->endpt_mgr.mac_endpts ))
&& (cnt < GC_MAX_LEAVE_NUM))
+ {
+ p_endpt = PARENT_STRUCT( p_item, ipoib_endpt_t, mac_item
);
+ p_item = cl_qmap_next( p_item );
+
+ /* Check if the current endpoint is not a multicast
listener */
+
+ if( p_endpt->h_mcast &&
+ (!p_endpt->is_mcast_listener) &&
+ ( cl_memcmp( &p_endpt->mac,
&DEFAULT_MCAST_GROUP, sizeof(mac_addr_t) ) &&
+ (!p_endpt->is_in_use) ))
+ {
+ cl_qmap_remove_item(
&p_port->endpt_mgr.mac_endpts,
+ &p_endpt->mac_item );
+ cl_fmap_remove_item(
&p_port->endpt_mgr.gid_endpts,
+ &p_endpt->gid_item );
+
+ if( p_endpt->dlid )
+ {
+ cl_qmap_remove_item(
&p_port->endpt_mgr.lid_endpts,
+ &p_endpt->lid_item );
+ p_endpt->dlid = 0;
+ }
+
+ cl_qlist_insert_tail(
+ &destroy_mc_list,
&p_endpt->mac_item.pool_item.list_item );
+ cnt++;
+ }
+ else
+ p_endpt->is_in_use = FALSE;
+ }
+ cl_obj_unlock( &p_port->obj );
+
+ /* Destroy all multicast endpoints now that we have released the
lock. */
+ while( cl_qlist_count( &destroy_mc_list ) )
+ {
+ p_endpt = PARENT_STRUCT( cl_qlist_head( &destroy_mc_list
),
+
ipoib_endpt_t, mac_item.pool_item.list_item );
+ IPOIB_PRINT( TRACE_LEVEL_INFORMATION, IPOIB_DBG_ENDPT,
+ ("mcast garbage collector: destroying endpoint
%02x:%02x:%02x:%02x:%02x:%02x \n",
+ p_endpt->mac.addr[0],
+ p_endpt->mac.addr[1],
+ p_endpt->mac.addr[2],
+ p_endpt->mac.addr[3],
+ p_endpt->mac.addr[4],
+ p_endpt->mac.addr[5]) );
+
+ cl_obj_destroy( &PARENT_STRUCT( cl_qlist_remove_head(
&destroy_mc_list ),
+ ipoib_endpt_t, mac_item.pool_item.list_item
)->obj );
+ }
+}
+
+static void __port_mcast_garbage_dpc(KDPC *p_gc_dpc,void *context,void
*s_arg1, void *s_arg2)
+{
+ ipoib_port_t *p_port = context;
+
+ UNREFERENCED_PARAMETER(p_gc_dpc);
+ UNREFERENCED_PARAMETER(s_arg1);
+ UNREFERENCED_PARAMETER(s_arg2);
+
+ __port_do_mcast_garbage(p_port);
+}
Index: ulp/ipoib/kernel/ipoib_port.h
===================================================================
--- ulp/ipoib/kernel/ipoib_port.h (revision 1394)
+++ ulp/ipoib/kernel/ipoib_port.h (working copy)
@@ -508,6 +508,8 @@
atomic32_t hdr_idx;
uint16_t pkey_index;
+ KDPC gc_dpc;
+ KTIMER gc_timer;
ipoib_hdr_t hdr[1]; /* Must be last!
*/
} ipoib_port_t;
Index: ulp/ipoib/kernel/netipoib.inx
===================================================================
--- ulp/ipoib/kernel/netipoib.inx (revision 1394)
+++ ulp/ipoib/kernel/netipoib.inx (working copy)
@@ -126,6 +126,12 @@
HKR, Ndi\Params\PayloadMtu, Min, 0, "60"
HKR, Ndi\Params\PayloadMtu, Max, 0, "4092"
+HKR, Ndi\Params\MCLeaveRescan, ParamDesc, 0, "MC leave
rescan (sec)"
+HKR, Ndi\Params\MCLeaveRescan, Type, 0, "dword"
+HKR, Ndi\Params\MCLeaveRescan, Default, 0, "260"
+HKR, Ndi\Params\MCLeaveRescan, Optional, 0, "0"
+HKR, Ndi\Params\MCLeaveRescan, Min, 0, "1"
+HKR, Ndi\Params\MCLeaveRescan, Max, 0, "3600"
[IpoibService]
DisplayName = %IpoibServiceDispName%
ServiceType = 1 ;%SERVICE_KERNEL_DRIVER%
-----Original Message-----
From: Fab Tillier [mailto:ftillier at windows.microsoft.com]
Sent: Wednesday, July 02, 2008 7:06 PM
To: Slava Strebkov; ofw at lists.openfabrics.org
Subject: RE: [ofw][patch] mcast garbage collector
>>You know, I think it would make a lot more sense to create the garbage
>>collector thread when you initialize the port object. It really
>>simplifies the error handling.
>
>Port needs garbage collector when it's in ACTIVE state. If port is
>disconnected, no need to keep garbage collector thread. That's why
>initialization and destruction are in port_up and port_down callbacks.
I'm just looking for ways to minimize error paths. Would using a timer
and DPC work? You wouldn't need to set the timer until the port becomes
active, and you can set the timer to be periodic using KeSetTimerEx.
You could then perform the garbage collection from the timer's DPC
callback (I didn't see any code in the garbage collection that would
prevent things from working at DISPATCH_LEVEL.)
-Fab
-------------- next part --------------
A non-text attachment was scrubbed...
Name: mcast_garbage_collector.diff
Type: application/octet-stream
Size: 8680 bytes
Desc: mcast_garbage_collector.diff
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20080714/03c9df09/attachment.obj>
More information about the ofw
mailing list