[openib-general] [PATCH] osm: handle local events
Eitan Zahavi
eitan at mellanox.co.il
Tue Aug 22 09:58:26 PDT 2006
I did not see this on the reflector.
We did have some mailer problems. So I am resending to the list
One more thing to add:
The only other event we considered was PORT_ACTIVE.
But as it turns out the event is only generated when the port moves into ACTIVE state
which means an SM already handled it...
EZ
Yevgeny Kliteynik wrote:
> Hi Hal
>
> This patch implements first item of the OSM todo list.
>
> OpenSM opens a thread that is listening for events on the SM's port.
> The events that are being taken care of are IBV_EVENT_DEVICE_FATAL and
> IBV_EVENT_PORT_ERROR.
>
> In case of IBV_EVENT_DEVICE_FATAL, osm is forced to exit.
> in case of IBV_EVENT_PORT_ERROR, osm initiates heavy sweep.
>
> Yevgeny
>
> Signed-off-by: Yevgeny Kliteynik <kliteyn at mellanox.co.il>
>
> Index: include/opensm/osm_sm_mad_ctrl.h
> ===================================================================
> --- include/opensm/osm_sm_mad_ctrl.h (revision 8998)
> +++ include/opensm/osm_sm_mad_ctrl.h (working copy)
> @@ -109,6 +109,7 @@ typedef struct _osm_sm_mad_ctrl
> osm_mad_pool_t *p_mad_pool;
> osm_vl15_t *p_vl15;
> osm_vendor_t *p_vendor;
> + struct _osm_state_mgr *p_state_mgr;
> osm_bind_handle_t h_bind;
> cl_plock_t *p_lock;
> cl_dispatcher_t *p_disp;
> @@ -130,6 +131,9 @@ typedef struct _osm_sm_mad_ctrl
> * p_vendor
> * Pointer to the vendor specific interfaces object.
> *
> +* p_state_mgr
> +* Pointer to the state manager object.
> +*
> * h_bind
> * Bind handle returned by the transport layer.
> *
> @@ -233,6 +237,7 @@ osm_sm_mad_ctrl_init(
> IN osm_mad_pool_t* const p_mad_pool,
> IN osm_vl15_t* const p_vl15,
> IN osm_vendor_t* const p_vendor,
> + IN struct _osm_state_mgr* const p_state_mgr,
> IN osm_log_t* const p_log,
> IN osm_stats_t* const p_stats,
> IN cl_plock_t* const p_lock,
> @@ -251,6 +256,9 @@ osm_sm_mad_ctrl_init(
> * p_vendor
> * [in] Pointer to the vendor specific interfaces object.
> *
> +* p_state_mgr
> +* [in] Pointer to the state manager object.
> +*
> * p_log
> * [in] Pointer to the log object.
> *
> Index: include/vendor/osm_vendor_ibumad.h
> ===================================================================
> --- include/vendor/osm_vendor_ibumad.h (revision 8998)
> +++ include/vendor/osm_vendor_ibumad.h (working copy)
> @@ -74,6 +74,8 @@ BEGIN_C_DECLS
> #define OSM_UMAD_MAX_CAS 32
> #define OSM_UMAD_MAX_PORTS_PER_CA 2
>
> +#define OSM_VENDOR_SUPPORT_EVENTS
> +
> /* OpenIB gen2 doesn't support RMPP yet */
>
> /****s* OpenSM: Vendor UMAD/osm_ca_info_t
> @@ -179,6 +181,10 @@ typedef struct _osm_vendor
> int umad_port_id;
> void *receiver;
> int issmfd;
> + cl_thread_t events_thread;
> + void * events_callback;
> + void * sm_context;
> + struct ibv_context * ibv_context;
> } osm_vendor_t;
>
> #define OSM_BIND_INVALID_HANDLE 0
> Index: include/vendor/osm_vendor_api.h
> ===================================================================
> --- include/vendor/osm_vendor_api.h (revision 8998)
> +++ include/vendor/osm_vendor_api.h (working copy)
> @@ -526,6 +526,110 @@ osm_vendor_set_debug(
> * SEE ALSO
> *********/
>
> +#ifdef OSM_VENDOR_SUPPORT_EVENTS
> +
> +#define OSM_EVENT_FATAL 1
> +#define OSM_EVENT_PORT_ERR 2
> +
> +/****s* OpenSM Vendor API/osm_vend_events_callback_t
> +* NAME
> +* osm_vend_events_callback_t
> +*
> +* DESCRIPTION
> +* Function prototype for the vendor events callback.
> +* The vendor layer calls this function on driver events.
> +*
> +* SYNOPSIS
> +*/
> +typedef void
> +(*osm_vend_events_callback_t)(
> + IN int events_mask,
> + IN void * const context );
> +/*
> +* PARAMETERS
> +* events_mask
> +* [in] The received event(s).
> +*
> +* context
> +* [in] Context supplied as the "sm_context" argument in
> +* the osm_vendor_unreg_events_cb call
> +*
> +* RETURN VALUES
> +* None.
> +*
> +* NOTES
> +*
> +* SEE ALSO
> +* osm_vendor_reg_events_cb osm_vendor_unreg_events_cb
> +*********/
> +
> +/****f* OpenSM Vendor API/osm_vendor_reg_events_cb
> +* NAME
> +* osm_vendor_reg_events_cb
> +*
> +* DESCRIPTION
> +* Registers the events callback function and start the events
> +* thread
> +*
> +* SYNOPSIS
> +*/
> +int
> +osm_vendor_reg_events_cb(
> + IN osm_vendor_t * const p_vend,
> + IN void * const sm_callback,
> + IN void * const sm_context);
> +/*
> +* PARAMETERS
> +* p_vend
> +* [in] vendor handle.
> +*
> +* sm_callback
> +* [in] Callback function that should be called when
> +* the event is received.
> +*
> +* sm_context
> +* [in] Context supplied as the "context" argument in
> +* the subsequenct calls to the sm_callback function
> +*
> +* RETURN VALUE
> +* IB_SUCCESS if OK.
> +*
> +* NOTES
> +*
> +* SEE ALSO
> +* osm_vend_events_callback_t osm_vendor_unreg_events_cb
> +*********/
> +
> +/****f* OpenSM Vendor API/osm_vendor_unreg_events_cb
> +* NAME
> +* osm_vendor_unreg_events_cb
> +*
> +* DESCRIPTION
> +* Un-Registers the events callback function and stops the events
> +* thread
> +*
> +* SYNOPSIS
> +*/
> +void
> +osm_vendor_unreg_events_cb(
> + IN osm_vendor_t * const p_vend);
> +/*
> +* PARAMETERS
> +* p_vend
> +* [in] vendor handle.
> +*
> +*
> +* RETURN VALUE
> +* None.
> +*
> +* NOTES
> +*
> +* SEE ALSO
> +* osm_vend_events_callback_t osm_vendor_reg_events_cb
> +*********/
> +
> +#endif /* OSM_VENDOR_SUPPORT_EVENTS */
> +
> END_C_DECLS
>
> #endif /* _OSM_VENDOR_API_H_ */
> Index: libvendor/osm_vendor_ibumad.c
> ===================================================================
> --- libvendor/osm_vendor_ibumad.c (revision 8998)
> +++ libvendor/osm_vendor_ibumad.c (working copy)
> @@ -72,6 +72,7 @@
> #include <opensm/osm_log.h>
> #include <opensm/osm_mad_pool.h>
> #include <vendor/osm_vendor_api.h>
> +#include <infiniband/verbs.h>
>
> /****s* OpenSM: Vendor AL/osm_umad_bind_info_t
> * NAME
> @@ -441,6 +442,91 @@ Exit:
>
> /**********************************************************************
> **********************************************************************/
> +static void
> +umad_events_thread(
> + IN void * vend_context)
> +{
> + int res = 0;
> + osm_vendor_t * p_vend = (osm_vendor_t *) vend_context;
> + struct ibv_async_event event;
> +
> + OSM_LOG_ENTER( p_vend->p_log, umad_events_thread );
> +
> + osm_log(p_vend->p_log, OSM_LOG_DEBUG,
> + "umad_events_thread: Device %s, async event FD: %d\n",
> + p_vend->umad_port.ca_name, p_vend->ibv_context->async_fd);
> + osm_log(p_vend->p_log, OSM_LOG_DEBUG,
> + "umad_events_thread: Listening for events on device %s, port %d\n",
> + p_vend->umad_port.ca_name, p_vend->umad_port.portnum);
> +
> + while (1) {
> +
> + res = ibv_get_async_event(p_vend->ibv_context, &event);
> + if (res)
> + {
> + osm_log(p_vend->p_log, OSM_LOG_ERROR,
> + "umad_events_thread: ERR 5450: "
> + "Failed getting async event (device %s, port %d)\n",
> + p_vend->umad_port.ca_name, p_vend->umad_port.portnum);
> + goto Exit;
> + }
> +
> + if (!p_vend->events_callback)
> + {
> + osm_log(p_vend->p_log, OSM_LOG_DEBUG,
> + "umad_events_thread: Events callback has been unregistered\n");
> + ibv_ack_async_event(&event);
> + goto Exit;
> + }
> + /*
> + * We're listening to events on the SM's port only
> + */
> + if ( event.element.port_num == p_vend->umad_port.portnum )
> + {
> + switch (event.event_type)
> + {
> + case IBV_EVENT_DEVICE_FATAL:
> + osm_log(p_vend->p_log, OSM_LOG_INFO,
> + "umad_events_thread: Received IBV_EVENT_DEVICE_FATAL\n");
> + ((osm_vend_events_callback_t)
> + (p_vend->events_callback))(OSM_EVENT_FATAL, p_vend->sm_context);
> +
> + ibv_ack_async_event(&event);
> + goto Exit;
> + break;
> +
> + case IBV_EVENT_PORT_ERR:
> + osm_log(p_vend->p_log, OSM_LOG_VERBOSE,
> + "umad_events_thread: Received IBV_EVENT_PORT_ERR\n");
> + ((osm_vend_events_callback_t)
> + (p_vend->events_callback))(OSM_EVENT_PORT_ERR, p_vend->sm_context);
> + break;
> +
> + default:
> + osm_log(p_vend->p_log, OSM_LOG_DEBUG,
> + "umad_events_thread: Received event #%d on port %d - Ignoring\n",
> + event.event_type, event.element.port_num);
> + }
> + }
> + else
> + {
> + osm_log(p_vend->p_log, OSM_LOG_DEBUG,
> + "umad_events_thread: Received event #%d on port %d - Ignoring\n",
> + event.event_type, event.element.port_num);
> + }
> +
> + ibv_ack_async_event(&event);
> + }
> +
> + Exit:
> + osm_log(p_vend->p_log, OSM_LOG_DEBUG,
> + "umad_events_thread: Terminating thread\n");
> + OSM_LOG_EXIT(p_vend->p_log);
> + return;
> +}
> +
> +/**********************************************************************
> + **********************************************************************/
> ib_api_status_t
> osm_vendor_init(
> IN osm_vendor_t* const p_vend,
> @@ -456,6 +542,7 @@ osm_vendor_init(
> p_vend->max_retries = OSM_DEFAULT_RETRY_COUNT;
> cl_spinlock_construct( &p_vend->cb_lock );
> cl_spinlock_construct( &p_vend->match_tbl_lock );
> + cl_thread_construct( &p_vend->events_thread );
> p_vend->umad_port_id = -1;
> p_vend->issmfd = -1;
>
> @@ -1217,4 +1304,114 @@ osm_vendor_set_debug(
> umad_debug(level);
> }
>
> +/**********************************************************************
> + **********************************************************************/
> +int
> +osm_vendor_reg_events_cb(
> + IN osm_vendor_t * const p_vend,
> + IN void * const sm_callback,
> + IN void * const sm_context)
> +{
> + ib_api_status_t status = IB_SUCCESS;
> + struct ibv_device ** dev_list;
> + struct ibv_device * device;
> +
> + OSM_LOG_ENTER( p_vend->p_log, osm_vendor_reg_events_cb );
> +
> + p_vend->events_callback = sm_callback;
> + p_vend->sm_context = sm_context;
> +
> + dev_list = ibv_get_device_list(NULL);
> + if (!dev_list || !(*dev_list)) {
> + osm_log(p_vend->p_log, OSM_LOG_ERROR,
> + "osm_vendor_reg_events_cb: ERR 5440: "
> + "No IB devices found\n");
> + status = IB_ERROR;
> + goto Exit;
> + }
> +
> + if (!p_vend->umad_port.ca_name || !p_vend->umad_port.ca_name[0])
> + {
> + osm_log(p_vend->p_log, OSM_LOG_ERROR,
> + "osm_vendor_reg_events_cb: ERR 5441: "
> + "Vendor initialization is not completed yet\n");
> + status = IB_ERROR;
> + goto Exit;
> + }
> +
> + osm_log(p_vend->p_log, OSM_LOG_DEBUG,
> + "osm_vendor_reg_events_cb: Registering on device %s\n",
> + p_vend->umad_port.ca_name);
> +
> + /*
> + * find device whos name matches the SM's device
> + */
> + for ( device = *dev_list;
> + (device != NULL) &&
> + (strcmp(p_vend->umad_port.ca_name, ibv_get_device_name(device)) != 0);
> + device += sizeof(struct ibv_device *) )
> + ;
> + if (!device)
> + {
> + osm_log(p_vend->p_log, OSM_LOG_ERROR,
> + "osm_vendor_reg_events_cb: ERR 5442: "
> + "Device %s hasn't been found in the device list\n"
> + ,p_vend->umad_port.ca_name);
> + status = IB_ERROR;
> + goto Exit;
> + }
> +
> + p_vend->ibv_context = ibv_open_device(device);
> + if (!p_vend->ibv_context) {
> + osm_log(p_vend->p_log, OSM_LOG_ERROR,
> + "osm_vendor_reg_events_cb: ERR 5443: "
> + "Couldn't get context for %s\n",
> + p_vend->umad_port.ca_name);
> + status = IB_ERROR;
> + goto Exit;
> + }
> +
> + /*
> + * Initiate the events thread
> + */
> + if (cl_thread_init(&p_vend->events_thread,
> + umad_events_thread,
> + p_vend,
> + "ibumad events thread") != CL_SUCCESS) {
> + osm_log(p_vend->p_log, OSM_LOG_ERROR,
> + "osm_vendor_reg_events_cb: ERR 5444: "
> + "Failed initiating event listening thread\n");
> + status = IB_ERROR;
> + goto Exit;
> + }
> +
> + Exit:
> + if (status != IB_SUCCESS)
> + {
> + p_vend->events_callback = NULL;
> + p_vend->sm_context = NULL;
> + p_vend->ibv_context = NULL;
> + p_vend->events_callback = NULL;
> + }
> + OSM_LOG_EXIT( p_vend->p_log );
> + return status;
> +}
> +
> +/**********************************************************************
> + **********************************************************************/
> +void
> +osm_vendor_unreg_events_cb(
> + IN osm_vendor_t * const p_vend)
> +{
> + OSM_LOG_ENTER( p_vend->p_log, osm_vendor_unreg_events_cb );
> + p_vend->events_callback = NULL;
> + p_vend->sm_context = NULL;
> + p_vend->ibv_context = NULL;
> + p_vend->events_callback = NULL;
> + OSM_LOG_EXIT( p_vend->p_log );
> +}
> +
> +/**********************************************************************
> + **********************************************************************/
> +
> #endif /* OSM_VENDOR_INTF_OPENIB */
> Index: libvendor/libosmvendor.map
> ===================================================================
> --- libvendor/libosmvendor.map (revision 8998)
> +++ libvendor/libosmvendor.map (working copy)
> @@ -1,4 +1,4 @@
> -OSMVENDOR_2.0 {
> +OSMVENDOR_2.1 {
> global:
> umad_receiver;
> osm_vendor_init;
> @@ -23,5 +23,7 @@ OSMVENDOR_2.0 {
> osmv_bind_sa;
> osmv_query_sa;
> osm_vendor_get_guid_ca_and_port;
> + osm_vendor_reg_events_cb;
> + osm_vendor_unreg_events_cb;
> local: *;
> };
> Index: opensm/osm_sm.c
> ===================================================================
> --- opensm/osm_sm.c (revision 8998)
> +++ opensm/osm_sm.c (working copy)
> @@ -313,6 +313,7 @@ osm_sm_init(
> p_sm->p_mad_pool,
> p_sm->p_vl15,
> p_sm->p_vendor,
> + &p_sm->state_mgr,
> p_log, p_stats, p_lock, p_disp );
> if( status != IB_SUCCESS )
> goto Exit;
> Index: opensm/osm_sm_mad_ctrl.c
> ===================================================================
> --- opensm/osm_sm_mad_ctrl.c (revision 8998)
> +++ opensm/osm_sm_mad_ctrl.c (working copy)
> @@ -59,6 +59,7 @@
> #include <opensm/osm_msgdef.h>
> #include <opensm/osm_helper.h>
> #include <opensm/osm_opensm.h>
> +#include <opensm/osm_state_mgr.h>
>
> /****f* opensm: SM/__osm_sm_mad_ctrl_retire_trans_mad
> * NAME
> @@ -953,6 +954,7 @@ osm_sm_mad_ctrl_init(
> IN osm_mad_pool_t* const p_mad_pool,
> IN osm_vl15_t* const p_vl15,
> IN osm_vendor_t* const p_vendor,
> + IN struct _osm_state_mgr* const p_state_mgr,
> IN osm_log_t* const p_log,
> IN osm_stats_t* const p_stats,
> IN cl_plock_t* const p_lock,
> @@ -969,6 +971,7 @@ osm_sm_mad_ctrl_init(
> p_ctrl->p_disp = p_disp;
> p_ctrl->p_mad_pool = p_mad_pool;
> p_ctrl->p_vendor = p_vendor;
> + p_ctrl->p_state_mgr = p_state_mgr;
> p_ctrl->p_stats = p_stats;
> p_ctrl->p_lock = p_lock;
> p_ctrl->p_vl15 = p_vl15;
> @@ -995,6 +998,47 @@ osm_sm_mad_ctrl_init(
>
> /**********************************************************************
> **********************************************************************/
> +void
> +__osm_vend_events_callback(
> + IN int events_mask,
> + IN void * const context )
> +{
> + osm_sm_mad_ctrl_t * const p_ctrl = (osm_sm_mad_ctrl_t * const) context;
> +
> + OSM_LOG_ENTER(p_ctrl->p_log, __osm_vend_events_callback);
> +
> + if (events_mask & OSM_EVENT_FATAL)
> + {
> + osm_log(p_ctrl->p_log, OSM_LOG_INFO,
> + "__osm_vend_events_callback: "
> + "Events callback got OSM_EVENT_FATAL\n");
> + osm_log(p_ctrl->p_log, OSM_LOG_SYS,
> + "Fatal HCA error - forcing OpenSM exit\n");
> + osm_exit_flag = 1;
> + OSM_LOG_EXIT(p_ctrl->p_log);
> + return;
> + }
> +
> + if (events_mask & OSM_EVENT_PORT_ERR)
> + {
> + osm_log(p_ctrl->p_log, OSM_LOG_INFO,
> + "__osm_vend_events_callback: "
> + "Events callback got OSM_EVENT_PORT_ERR - forcing heavy sweep\n");
> + p_ctrl->p_subn->force_immediate_heavy_sweep = TRUE;
> + osm_state_mgr_process((osm_state_mgr_t * const)p_ctrl->p_state_mgr,
> + OSM_SIGNAL_SWEEP);
> + OSM_LOG_EXIT(p_ctrl->p_log);
> + return;
> + }
> +
> + osm_log(p_ctrl->p_log, OSM_LOG_INFO,
> + "__osm_vend_events_callback: "
> + "Events callback got event mask of %d - No action taken\n");
> + OSM_LOG_EXIT(p_ctrl->p_log);
> +}
> +
> +/**********************************************************************
> + **********************************************************************/
> ib_api_status_t
> osm_sm_mad_ctrl_bind(
> IN osm_sm_mad_ctrl_t* const p_ctrl,
> @@ -1044,6 +1088,17 @@ osm_sm_mad_ctrl_bind(
> goto Exit;
> }
>
> + if ( osm_vendor_reg_events_cb(p_ctrl->p_vendor,
> + __osm_vend_events_callback,
> + p_ctrl) )
> + {
> + status = IB_ERROR;
> + osm_log( p_ctrl->p_log, OSM_LOG_ERROR,
> + "osm_sm_mad_ctrl_bind: ERR 3120: "
> + "Vendor failed to register for events\n" );
> + goto Exit;
> + }
> +
> Exit:
> OSM_LOG_EXIT( p_ctrl->p_log );
> return( status );
> Index: config/osmvsel.m4
> ===================================================================
> --- config/osmvsel.m4 (revision 8998)
> +++ config/osmvsel.m4 (working copy)
> @@ -63,9 +63,9 @@ if test $with_osmv = "openib"; then
> OSMV_CFLAGS="-DOSM_VENDOR_INTF_OPENIB"
> OSMV_INCLUDES="-I\$(srcdir)/../include -I\$(srcdir)/../../libibcommon/include/infiniband -I\$(srcdir)/../../libibumad/include/infiniband"
> if test "x$with_umad_libs" = "x"; then
> - OSMV_LDADD="-libumad"
> + OSMV_LDADD="-libumad -libverbs"
> else
> - OSMV_LDADD="-L$with_umad_libs -libumad"
> + OSMV_LDADD="-L$with_umad_libs -libumad -libverbs"
> fi
>
> if test "x$with_umad_includes" != "x"; then
> @@ -137,6 +137,8 @@ if test "$disable_libcheck" != "yes"; th
> LDFLAGS="$LDFLAGS $OSMV_LDADD"
> AC_CHECK_LIB(ibumad, umad_init, [],
> AC_MSG_ERROR([umad_init() not found. libosmvendor of type openib requires libibumad.]))
> + AC_CHECK_LIB(ibverbs, ibv_get_device_list, [],
> + AC_MSG_ERROR([umad_init() not found. libosmvendor of type openib requires libibverbs.]))
> LD_FLAGS=$osmv_save_ldflags
> elif test $with_osmv = "sim" ; then
> LDFLAGS="$LDFLAGS -L$with_sim/lib"
>
More information about the general
mailing list