[ofw] patch: [ibbus] Add A new function to IBAL that allows one to create a multicast group without attaching a QP to it.
Tzachi Dar
tzachid at mellanox.co.il
Sun Feb 20 00:57:03 PST 2011
Applied on 3096.
Thanks
Tzachi
> -----Original Message-----
> From: Tzachi Dar
> Sent: Thursday, February 17, 2011 11:01 AM
> To: Tzachi Dar; Hefty, Sean; ofw at lists.openfabrics.org
> Subject: RE: [ofw] patch: [ibbus] Add A new function to IBAL that allows one
> to create a multicast group without attaching a QP to it.
>
> Attached is the latest version of the patch.
>
> The mcast object is now also an out parameter (optional). One can attach it to
> a different object or track it by some other mechanism. Please note that since
> the ib_mcast_handle_t is a user object there is no need for the kernel to keep
> tracking it.
>
> Thanks
> Tzachi & Irena
>
>
> Index: core/al/al_mcast.c
> ===================================================================
> --- core/al/al_mcast.c (revision 3095)
> +++ core/al/al_mcast.c (working copy)
> @@ -100,11 +100,11 @@
> #endif
>
>
> -
> ib_api_status_t
> -al_join_mcast(
> - IN const ib_qp_handle_t h_qp,
> - IN const ib_mcast_req_t* const p_mcast_req )
> +al_join_mcast_common(
> + IN const ib_qp_handle_t h_qp
> OPTIONAL,
> + IN const ib_mcast_req_t* const p_mcast_req,
> + OUT ib_mcast_handle_t* p_h_mcast
> )
> {
> ib_mcast_handle_t h_mcast;
> ib_api_status_t status;
> @@ -113,20 +113,6 @@
>
> AL_ENTER( AL_DBG_MCAST );
>
> - /*
> - * Validate the port GUID. There is no need to validate the pkey index
> as
> - * the user could change it later to make it invalid. There is also no
> - * need to perform any QP transitions as ib_init_dgrm_svc resets the QP
> and
> - * starts from scratch.
> - */
> - status = get_port_num( h_qp->obj.p_ci_ca, p_mcast_req->port_guid, NULL
> );
> - if( status != IB_SUCCESS )
> - {
> - AL_PRINT( TRACE_LEVEL_ERROR, AL_DBG_ERROR,
> - ("get_port_num failed, status: %s\n",
> ib_get_err_str(status)) );
> - return status;
> - }
> -
> /* Allocate a new multicast request. */
> h_mcast = cl_zalloc( sizeof( ib_mcast_t ) );
> if( !h_mcast )
> @@ -183,13 +169,16 @@
> h_mcast->port_guid = p_mcast_req->port_guid;
>
> /* Track the multicast with the QP instance. */
> - status = attach_al_obj( &h_qp->obj, &h_mcast->obj );
> - if( status != IB_SUCCESS )
> - {
> - h_mcast->obj.pfn_destroy( &h_mcast->obj, NULL );
> - AL_PRINT_EXIT( TRACE_LEVEL_ERROR, AL_DBG_ERROR,
> - ("attach_al_obj returned %s.\n", ib_get_err_str(status))
> );
> - return status;
> +
> + if (h_qp) {
> + status = attach_al_obj( &h_qp->obj, &h_mcast->obj );
> + if( status != IB_SUCCESS )
> + {
> + h_mcast->obj.pfn_destroy( &h_mcast->obj, NULL );
> + AL_PRINT_EXIT( TRACE_LEVEL_ERROR, AL_DBG_ERROR,
> + ("attach_al_obj returned %s.\n",
> ib_get_err_str(status)) );
> + return status;
> + }
> }
>
> /* Issue the MAD to the SA. */
> @@ -214,6 +203,12 @@
> h_mcast->obj.pfn_destroy( &h_mcast->obj, NULL );
> }
>
> + if (p_h_mcast)
> + {
> + ref_al_obj(&h_mcast->obj);
> + *p_h_mcast = h_mcast;
> + }
> +
> /*
> * Note: Don't release the reference taken in init_al_obj while we
> * have the SA req outstanding.
> @@ -224,6 +219,40 @@
> }
>
>
> +ib_api_status_t
> +al_join_mcast_no_qp(
> + IN const ib_mcast_req_t* const p_mcast_req,
> + OUT ib_mcast_handle_t* p_h_mcast
> )
> +{
> + return al_join_mcast_common(NULL, p_mcast_req, p_h_mcast );
> +
> +}
> +
> +ib_api_status_t
> +al_join_mcast(
> + IN const ib_qp_handle_t h_qp,
> + IN const ib_mcast_req_t* const p_mcast_req,
> + OUT ib_mcast_handle_t* p_h_mcast
> )
> +{
> + ib_api_status_t status;
> + /*
> + * Validate the port GUID. There is no need to validate the pkey index
> as
> + * the user could change it later to make it invalid. There is also no
> + * need to perform any QP transitions as ib_init_dgrm_svc resets the QP
> and
> + * starts from scratch.
> + */
> +
> + status = get_port_num( h_qp->obj.p_ci_ca, p_mcast_req->port_guid, NULL
> );
> + if( status != IB_SUCCESS )
> + {
> + AL_PRINT( TRACE_LEVEL_ERROR, AL_DBG_ERROR,
> + ("get_port_num failed, status: %s\n",
> ib_get_err_str(status)) );
> + return status;
> + }
> +
> + return al_join_mcast_common(h_qp, p_mcast_req, p_h_mcast); }
> +
> static void
> __destroying_mcast(
> IN al_obj_t
> *p_obj )
> @@ -504,7 +533,8 @@
> */
> h_mcast->state = SA_REG_ACTIVE;
> /* Attach the QP to the multicast group. */
> - if(ib_member_get_state(mcast_rec.p_member_rec-
> >scope_state) == IB_MC_REC_STATE_FULL_MEMBER)
> + if(ib_member_get_state(mcast_rec.p_member_rec-
> >scope_state) == IB_MC_REC_STATE_FULL_MEMBER &&
> + (((ib_qp_handle_t)h_mcast->obj.p_parent_obj) !=
> NULL))
> {
> status = verbs_attach_mcast(h_mcast);
> if( status != IB_SUCCESS )
> Index: core/al/al_mcast.h
> ===================================================================
> --- core/al/al_mcast.h (revision 3095)
> +++ core/al/al_mcast.h (working copy)
> @@ -94,9 +94,15 @@
> ib_api_status_t
> al_join_mcast(
> IN const ib_qp_handle_t h_qp,
> - IN const ib_mcast_req_t* const p_mcast_req );
> + IN const ib_mcast_req_t* const p_mcast_req,
> + OUT ib_mcast_handle_t* p_h_mcast
> );
>
> +ib_api_status_t
> +al_join_mcast_no_qp(
> + IN const ib_mcast_req_t* const p_mcast_req,
> + OUT ib_mcast_handle_t* p_h_mcast
> );
>
> +
> #if defined( CL_KERNEL )
> /*
> * Called by proxy to attach a QP to a multicast group.
> @@ -109,6 +115,7 @@
> OUT al_attach_handle_t *ph_attach,
> IN OUT ci_umv_buf_t *p_umv_buf
> OPTIONAL );
>
> +
> #endif /* CL_KERNEL */
>
>
> Index: core/al/al_qp.c
> ===================================================================
> --- core/al/al_qp.c (revision 3095)
> +++ core/al/al_qp.c (working copy)
> @@ -597,10 +597,12 @@
> static ib_api_status_t
> al_bad_join_mcast(
> IN const ib_qp_handle_t h_qp,
> - IN const ib_mcast_req_t* const p_mcast_req )
> + IN const ib_mcast_req_t* const p_mcast_req,
> + OUT ib_mcast_handle_t* p_h_mcast
> )
> {
> UNUSED_PARAM( h_qp );
> UNUSED_PARAM( p_mcast_req );
> + UNUSED_PARAM( p_h_mcast );
> return IB_INVALID_PARAMETER;
> }
>
> @@ -1571,7 +1573,8 @@
> ib_api_status_t
> ib_join_mcast(
> IN const ib_qp_handle_t h_qp,
> - IN const ib_mcast_req_t* const p_mcast_req )
> + IN const ib_mcast_req_t* const p_mcast_req,
> + OUT ib_mcast_handle_t* p_h_mcast
> )
> {
> ib_api_status_t status;
>
> @@ -1588,14 +1591,36 @@
> return IB_INVALID_PARAMETER;
> }
>
> - status = h_qp->pfn_join_mcast( h_qp, p_mcast_req );
> + status = h_qp->pfn_join_mcast( h_qp, p_mcast_req, p_h_mcast );
>
> AL_EXIT( AL_DBG_MCAST );
> return status;
> }
>
>
> +ib_api_status_t
> +ib_join_mcast_no_qp(
> + IN const ib_mcast_req_t* const p_mcast_req,
> + OUT ib_mcast_handle_t* p_h_mcast
> )
> +{
> + ib_api_status_t status;
>
> + AL_ENTER( AL_DBG_MCAST );
> +
> + if( !p_mcast_req )
> + {
> + AL_PRINT_EXIT( TRACE_LEVEL_ERROR, AL_DBG_ERROR,
> ("IB_INVALID_PARAMETER\n") );
> + return IB_INVALID_PARAMETER;
> + }
> +
> + status = al_join_mcast_no_qp( p_mcast_req, p_h_mcast);
> +
> + AL_EXIT( AL_DBG_MCAST );
> + return status;
> +}
> +
> +
> +
> /*
> * Post a work request to the send queue of the QP.
> */
> Index: core/al/al_qp.h
> ===================================================================
> --- core/al/al_qp.h (revision 3095)
> +++ core/al/al_qp.h (working copy)
> @@ -133,7 +133,8 @@
> ib_api_status_t
> (*pfn_join_mcast)(
> IN const ib_qp_handle_t h_qp,
> - IN const ib_mcast_req_t* const p_mcast_req
> );
> + IN const ib_mcast_req_t* const p_mcast_req,
> + OUT ib_mcast_handle_t*
> p_h_mcast );
>
> } ib_qp_t;
>
> Index: core/bus/kernel/bus_pnp.c
> ===================================================================
> --- core/bus/kernel/bus_pnp.c (revision 3095)
> +++ core/bus/kernel/bus_pnp.c (working copy)
> @@ -53,6 +53,10 @@
> #include "iba/ib_al_ifc.h"
> #include "iba/ib_ci_ifc.h"
> #include "iba/ib_cm_ifc.h"
> +#include "al_cm_cep.h"
> +#include "al_mgr.h"
> +#include "bus_ev_log.h"
> +#include "al_proxy.h"
>
>
> /* Interface names are generated by IoRegisterDeviceInterface. */ @@ -1012,6
> +1016,7 @@
> p_ifc->poll_cq = ib_poll_cq;
> p_ifc->rearm_cq = ib_rearm_cq;
> p_ifc->join_mcast = ib_join_mcast;
> + p_ifc->join_mcast_no_qp = al_join_mcast_no_qp;
> p_ifc->leave_mcast = ib_leave_mcast;
> p_ifc->local_mad = ib_local_mad;
> p_ifc->cm_listen = ib_cm_listen;
> Index: inc/iba/ib_al.h
> ===================================================================
> --- inc/iba/ib_al.h (revision 3095)
> +++ inc/iba/ib_al.h (working copy)
> @@ -3797,14 +3797,15 @@
> * ib_join_mcast
> *
> * DESCRIPTION
> -* Attaches a queue pair to a multicast group.
> +* Creates a multicast group and Attaches a queue pair to it.
> *
> * SYNOPSIS
> */
> AL_EXPORT ib_api_status_t AL_API
> ib_join_mcast(
> IN const ib_qp_handle_t h_qp,
> - IN const ib_mcast_req_t* const p_mcast_req );
> + IN const ib_mcast_req_t* const p_mcast_req,
> + OUT ib_mcast_handle_t* p_h_mcast
> );
> /*
> * PARAMETERS
> * h_qp
> @@ -3872,7 +3873,85 @@
> * ib_leave_mcast, ib_mcast_req_t, ib_create_qp, ib_init_dgrm_svc
> *****/
>
> +/****f* Access Layer/ib_join_mcast
> +* NAME
> +* ib_join_mcast_no_qp
> +*
> +* DESCRIPTION
> +* Creates a multicast group.
> +*
> +* SYNOPSIS
> +*/
> +AL_EXPORT ib_api_status_t AL_API
> +ib_join_mcast_no_qp(
> + IN const ib_mcast_req_t* const p_mcast_req,
> + OUT ib_mcast_handle_t* p_h_mcast
> );
> +/*
> +* PARAMETERS
> +*
> +* p_mcast_req
> +* [in] Specifies the multicast group to join.
> +*
> +* RETURN VALUES
> +* IB_SUCCESS
> +* The join multicast group request has been initiated.
> +*
> +* IB_INVALID_QP_HANDLE
> +* The queue pair handle was invalid.
> +*
> +* IB_INVALID_PARAMETER
> +* A reference to the multicast group request information was not
> +* provided.
> +*
> +* IB_INVALID_SERVICE_TYPE
> +* The queue pair configuration does not support this type of
> service.
> +*
> +* IB_INSUFFICIENT_MEMORY
> +* There was insufficient memory to join the multicast group.
> +*
> +* IB_INVALID_GUID
> +* No port was found for the port_guid specified in the request.
> +*
> +* IB_INSUFFICIENT_RESOURCES
> +* There were insufficient resources currently available on the
> channel
> +* adapter to perform the operation.
> +*
> +* IB_INVALID_PKEY
> +* The pkey specified in the multicast join request does not match
> the
> +* pkey of the queue pair.
> +*
> +* IB_INVALID_PORT
> +* The port GUID specified in the multicast join request does not
> match
> +* the port of the queue pair.
> +*
> +* IB_ERROR
> +* An error occurred while performing the multicast group join
> operation.
> +*
> +* IB_INSUFFICIENT_RESOURCES
> +* There were insufficient resources currently available to complete
> +* the request.
> +*
> +* IB_INSUFFICIENT_MEMORY
> +* There was insufficient memory to complete the request.
> +*
> +* NOTES
> +* This routine results in the specified queue pair joining a multicast
> +* group. If the multicast group does not already exist, it will be
> created
> +* at the user's option. Information about the multicast group is returned
> +* to the user through a callback specified through the p_mcast_req
> +* parameter.
> +*
> +* If the specified queue pair is already a member of a multicast group
> when
> +* this call is invoked, an error will occur if there are conflicting
> +* membership requirements. The QP is restricted to being bound to a
> single
> +* port_guid and using a single pkey.
> +*
> +* SEE ALSO
> +* ib_leave_mcast, ib_mcast_req_t, ib_create_qp, ib_init_dgrm_svc
> +*****/
>
> +
> +
> /****f* Access Layer/ib_leave_mcast
> * NAME
> * ib_leave_mcast
> Index: inc/kernel/iba/ib_al_ifc.h
> ===================================================================
> --- inc/kernel/iba/ib_al_ifc.h (revision 3095)
> +++ inc/kernel/iba/ib_al_ifc.h (working copy)
> @@ -48,7 +48,7 @@
> * IB resources provided by HCAs.
> *********/
>
> -#define AL_INTERFACE_VERSION (13)
> +#define AL_INTERFACE_VERSION (15)
>
>
>
> @@ -401,9 +401,15 @@
> typedef ib_api_status_t
> (*ib_pfn_join_mcast_t)(
> IN const ib_qp_handle_t h_qp,
> - IN const ib_mcast_req_t* const p_mcast_req );
> + IN const ib_mcast_req_t* const p_mcast_req,
> + OUT ib_mcast_handle_t* p_h_mcast
> );
>
> typedef ib_api_status_t
> +(*ib_pfn_join_mcast_no_qp_t)(
> + IN const ib_mcast_req_t* const p_mcast_req,
> + OUT ib_mcast_handle_t* p_h_mcast
> );
> +
> +typedef ib_api_status_t
> (*ib_pfn_leave_mcast_t)(
> IN const ib_mcast_handle_t h_mcast,
> IN const ib_pfn_destroy_cb_t destroy_cb );
> @@ -702,6 +708,7 @@
> ib_pfn_poll_cq_t poll_cq;
> ib_pfn_rearm_cq_t rearm_cq;
> ib_pfn_join_mcast_t join_mcast;
> + ib_pfn_join_mcast_no_qp_t join_mcast_no_qp;
> ib_pfn_leave_mcast_t leave_mcast;
> ib_pfn_local_mad_t local_mad;
> ib_pfn_cm_listen_t cm_listen;
> Index: ulp/ipoib/kernel/ipoib_port.c
> ===================================================================
> --- ulp/ipoib/kernel/ipoib_port.c (revision 3095)
> +++ ulp/ipoib/kernel/ipoib_port.c (working copy)
> @@ -5634,7 +5634,7 @@
> ipoib_port_ref( p_port, ref_join_bcast );
>
> status = p_port->p_adapter->p_ifc->join_mcast(
> - p_port->ib_mgr.h_qp, &mcast_req );
> + p_port->ib_mgr.h_qp, &mcast_req, NULL );
> if( status != IB_SUCCESS )
> {
> ipoib_port_deref( p_port, ref_bcast_join_failed ); @@ -5695,7
> +5695,7 @@
> /* reference the object for the multicast join request. */
> ipoib_port_ref( p_port, ref_join_bcast );
>
> - status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp,
> &mcast_req );
> + status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp,
> +&mcast_req, NULL );
> if( status != IB_SUCCESS )
> {
> ipoib_port_deref( p_port, ref_bcast_create_failed ); @@ -6155,7
> +6155,7 @@
> /* reference the object for the multicast join request. */
> ipoib_port_ref( p_port, ref_join_mcast );
>
> - status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp,
> &mcast_req );
> + status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp,
> +&mcast_req, NULL );
> if( status != IB_SUCCESS )
> {
> ipoib_port_deref( p_port, ref_mcast_join_failed );
> Index: ulp/ipoib_NDIS6_CM/kernel/ipoib_port.cpp
> ===================================================================
> --- ulp/ipoib_NDIS6_CM/kernel/ipoib_port.cpp (revision 3095)
> +++ ulp/ipoib_NDIS6_CM/kernel/ipoib_port.cpp (working copy)
> @@ -8327,7 +8327,7 @@
> /* reference the object for the multicast join request. */
> ipoib_port_ref( p_port, ref_join_bcast );
>
> - status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp,
> &mcast_req );
> + status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp,
> +&mcast_req, NULL );
> if( status != IB_SUCCESS )
> {
> ipoib_port_deref( p_port, ref_bcast_join_failed ); @@ -8388,7
> +8388,7 @@
> /* reference the object for the multicast join request. */
> ipoib_port_ref( p_port, ref_join_bcast );
>
> - status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp,
> &mcast_req );
> + status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp,
> +&mcast_req, NULL );
> if( status != IB_SUCCESS )
> {
> ipoib_port_deref( p_port, ref_bcast_create_failed ); @@ -8908,7
> +8908,7 @@
> /* reference the object for the multicast join request. */
> ipoib_port_ref( p_port, ref_join_mcast );
>
> - status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp,
> &mcast_req );
> + status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp,
> +&mcast_req, NULL );
> if( status != IB_SUCCESS )
> {
> ipoib_port_deref( p_port, ref_mcast_join_failed );
>
> > -----Original Message-----
> > From: ofw-bounces at lists.openfabrics.org [mailto:ofw-
> > bounces at lists.openfabrics.org] On Behalf Of Tzachi Dar
> > Sent: Tuesday, February 15, 2011 11:45 PM
> > To: Hefty, Sean; ofw at lists.openfabrics.org
> > Subject: Re: [ofw] patch: [ibbus] Add A new function to IBAL that
> > allows one to create a multicast group without attaching a QP to it.
> >
> > Hi Sean,
> >
> > I now see why you need the h_mcast as an output parameter. We will add
> > this to the function, but I don't think we will add the cancel
> > multicast on the first stage.
> >
> > We will also see how we can track the user objects from the kernel.
> >
> > Thanks
> > Tzachi
> >
> > ________________________________________
> > From: Hefty, Sean [sean.hefty at intel.com]
> > Sent: Tuesday, February 15, 2011 8:07 PM
> > To: Tzachi Dar; ofw at lists.openfabrics.org
> > Subject: RE: [ofw] patch: [ibbus] Add A new function to IBAL that
> > allows one to create a multicast group without attaching a QP to it.
> >
> > > 1) On ib_join_mcast( ..., ib_mcast_handle_t *h_mcast); the h_mcast
> > > gets back to the user from the call back.
> > > I guess that there is no problem to return the h_mcast in the call
> > > itself but are we sure this has a good reason?
> >
> > This allows the user to cancel the join operation without destroying
> > everything by closing their al instance. For example, after issuing a
> > join, they can immediately respond to an SM change, reregister, or
> > other event by freeing the join and issuing a new one. Without this,
> > handling those events becomes more difficult.
> >
> > Hmm... on the Linux side, the multicast module reports those types of
> > events against the join request. The user can receive multiple
> > (serialized) callbacks for a single join call. How does ibal handle
> > errors on a multicast group after a successful join?
> >
> >
> > > 2) " Personally, I'm fine requiring the caller to handle the cleanup
> > > (provided the kernel cleans up after user space)."
> > > Doesn't this has an influence on the API which would actually force
> > > us to have something else in the API (for example h_al) or something
> similar.
> >
> > Not necessarily. The user to kernel proxy code needs to associate the
> > h_mcast with another object, such as the file, but that doesn't have
> > to be in the kernel API. The proxy code must free the multicast
> > object when the file is closed. You can look at the winverbs kernel cleanup
> code as an example.
> >
> > - Sean
> > _______________________________________________
> > ofw mailing list
> > ofw at lists.openfabrics.org
> > http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ofw
More information about the ofw
mailing list