[ofw] patch: [ibbus] Add A new function to IBAL that allows one to create a multicast group without attaching a QP to it.
Tzachi Dar
tzachid at mellanox.co.il
Thu Feb 17 01:00:57 PST 2011
Attached is the latest version of the patch.
The mcast object is now also an out parameter (optional). One can attach it to a different object or track it by some other mechanism. Please note that since the ib_mcast_handle_t is a user object there is no need for the kernel to keep tracking it.
Thanks
Tzachi & Irena
Index: core/al/al_mcast.c
===================================================================
--- core/al/al_mcast.c (revision 3095)
+++ core/al/al_mcast.c (working copy)
@@ -100,11 +100,11 @@
#endif
-
ib_api_status_t
-al_join_mcast(
- IN const ib_qp_handle_t h_qp,
- IN const ib_mcast_req_t* const p_mcast_req )
+al_join_mcast_common(
+ IN const ib_qp_handle_t h_qp OPTIONAL,
+ IN const ib_mcast_req_t* const p_mcast_req,
+ OUT ib_mcast_handle_t* p_h_mcast )
{
ib_mcast_handle_t h_mcast;
ib_api_status_t status;
@@ -113,20 +113,6 @@
AL_ENTER( AL_DBG_MCAST );
- /*
- * Validate the port GUID. There is no need to validate the pkey index as
- * the user could change it later to make it invalid. There is also no
- * need to perform any QP transitions as ib_init_dgrm_svc resets the QP and
- * starts from scratch.
- */
- status = get_port_num( h_qp->obj.p_ci_ca, p_mcast_req->port_guid, NULL );
- if( status != IB_SUCCESS )
- {
- AL_PRINT( TRACE_LEVEL_ERROR, AL_DBG_ERROR,
- ("get_port_num failed, status: %s\n", ib_get_err_str(status)) );
- return status;
- }
-
/* Allocate a new multicast request. */
h_mcast = cl_zalloc( sizeof( ib_mcast_t ) );
if( !h_mcast )
@@ -183,13 +169,16 @@
h_mcast->port_guid = p_mcast_req->port_guid;
/* Track the multicast with the QP instance. */
- status = attach_al_obj( &h_qp->obj, &h_mcast->obj );
- if( status != IB_SUCCESS )
- {
- h_mcast->obj.pfn_destroy( &h_mcast->obj, NULL );
- AL_PRINT_EXIT( TRACE_LEVEL_ERROR, AL_DBG_ERROR,
- ("attach_al_obj returned %s.\n", ib_get_err_str(status)) );
- return status;
+
+ if (h_qp) {
+ status = attach_al_obj( &h_qp->obj, &h_mcast->obj );
+ if( status != IB_SUCCESS )
+ {
+ h_mcast->obj.pfn_destroy( &h_mcast->obj, NULL );
+ AL_PRINT_EXIT( TRACE_LEVEL_ERROR, AL_DBG_ERROR,
+ ("attach_al_obj returned %s.\n", ib_get_err_str(status)) );
+ return status;
+ }
}
/* Issue the MAD to the SA. */
@@ -214,6 +203,12 @@
h_mcast->obj.pfn_destroy( &h_mcast->obj, NULL );
}
+ if (p_h_mcast)
+ {
+ ref_al_obj(&h_mcast->obj);
+ *p_h_mcast = h_mcast;
+ }
+
/*
* Note: Don't release the reference taken in init_al_obj while we
* have the SA req outstanding.
@@ -224,6 +219,40 @@
}
+ib_api_status_t
+al_join_mcast_no_qp(
+ IN const ib_mcast_req_t* const p_mcast_req,
+ OUT ib_mcast_handle_t* p_h_mcast )
+{
+ return al_join_mcast_common(NULL, p_mcast_req, p_h_mcast );
+
+}
+
+ib_api_status_t
+al_join_mcast(
+ IN const ib_qp_handle_t h_qp,
+ IN const ib_mcast_req_t* const p_mcast_req,
+ OUT ib_mcast_handle_t* p_h_mcast )
+{
+ ib_api_status_t status;
+ /*
+ * Validate the port GUID. There is no need to validate the pkey index as
+ * the user could change it later to make it invalid. There is also no
+ * need to perform any QP transitions as ib_init_dgrm_svc resets the QP and
+ * starts from scratch.
+ */
+
+ status = get_port_num( h_qp->obj.p_ci_ca, p_mcast_req->port_guid, NULL );
+ if( status != IB_SUCCESS )
+ {
+ AL_PRINT( TRACE_LEVEL_ERROR, AL_DBG_ERROR,
+ ("get_port_num failed, status: %s\n", ib_get_err_str(status)) );
+ return status;
+ }
+
+ return al_join_mcast_common(h_qp, p_mcast_req, p_h_mcast);
+}
+
static void
__destroying_mcast(
IN al_obj_t *p_obj )
@@ -504,7 +533,8 @@
*/
h_mcast->state = SA_REG_ACTIVE;
/* Attach the QP to the multicast group. */
- if(ib_member_get_state(mcast_rec.p_member_rec->scope_state) == IB_MC_REC_STATE_FULL_MEMBER)
+ if(ib_member_get_state(mcast_rec.p_member_rec->scope_state) == IB_MC_REC_STATE_FULL_MEMBER &&
+ (((ib_qp_handle_t)h_mcast->obj.p_parent_obj) != NULL))
{
status = verbs_attach_mcast(h_mcast);
if( status != IB_SUCCESS )
Index: core/al/al_mcast.h
===================================================================
--- core/al/al_mcast.h (revision 3095)
+++ core/al/al_mcast.h (working copy)
@@ -94,9 +94,15 @@
ib_api_status_t
al_join_mcast(
IN const ib_qp_handle_t h_qp,
- IN const ib_mcast_req_t* const p_mcast_req );
+ IN const ib_mcast_req_t* const p_mcast_req,
+ OUT ib_mcast_handle_t* p_h_mcast );
+ib_api_status_t
+al_join_mcast_no_qp(
+ IN const ib_mcast_req_t* const p_mcast_req,
+ OUT ib_mcast_handle_t* p_h_mcast );
+
#if defined( CL_KERNEL )
/*
* Called by proxy to attach a QP to a multicast group.
@@ -109,6 +115,7 @@
OUT al_attach_handle_t *ph_attach,
IN OUT ci_umv_buf_t *p_umv_buf OPTIONAL );
+
#endif /* CL_KERNEL */
Index: core/al/al_qp.c
===================================================================
--- core/al/al_qp.c (revision 3095)
+++ core/al/al_qp.c (working copy)
@@ -597,10 +597,12 @@
static ib_api_status_t
al_bad_join_mcast(
IN const ib_qp_handle_t h_qp,
- IN const ib_mcast_req_t* const p_mcast_req )
+ IN const ib_mcast_req_t* const p_mcast_req,
+ OUT ib_mcast_handle_t* p_h_mcast )
{
UNUSED_PARAM( h_qp );
UNUSED_PARAM( p_mcast_req );
+ UNUSED_PARAM( p_h_mcast );
return IB_INVALID_PARAMETER;
}
@@ -1571,7 +1573,8 @@
ib_api_status_t
ib_join_mcast(
IN const ib_qp_handle_t h_qp,
- IN const ib_mcast_req_t* const p_mcast_req )
+ IN const ib_mcast_req_t* const p_mcast_req,
+ OUT ib_mcast_handle_t* p_h_mcast )
{
ib_api_status_t status;
@@ -1588,14 +1591,36 @@
return IB_INVALID_PARAMETER;
}
- status = h_qp->pfn_join_mcast( h_qp, p_mcast_req );
+ status = h_qp->pfn_join_mcast( h_qp, p_mcast_req, p_h_mcast );
AL_EXIT( AL_DBG_MCAST );
return status;
}
+ib_api_status_t
+ib_join_mcast_no_qp(
+ IN const ib_mcast_req_t* const p_mcast_req,
+ OUT ib_mcast_handle_t* p_h_mcast )
+{
+ ib_api_status_t status;
+ AL_ENTER( AL_DBG_MCAST );
+
+ if( !p_mcast_req )
+ {
+ AL_PRINT_EXIT( TRACE_LEVEL_ERROR, AL_DBG_ERROR, ("IB_INVALID_PARAMETER\n") );
+ return IB_INVALID_PARAMETER;
+ }
+
+ status = al_join_mcast_no_qp( p_mcast_req, p_h_mcast);
+
+ AL_EXIT( AL_DBG_MCAST );
+ return status;
+}
+
+
+
/*
* Post a work request to the send queue of the QP.
*/
Index: core/al/al_qp.h
===================================================================
--- core/al/al_qp.h (revision 3095)
+++ core/al/al_qp.h (working copy)
@@ -133,7 +133,8 @@
ib_api_status_t
(*pfn_join_mcast)(
IN const ib_qp_handle_t h_qp,
- IN const ib_mcast_req_t* const p_mcast_req );
+ IN const ib_mcast_req_t* const p_mcast_req,
+ OUT ib_mcast_handle_t* p_h_mcast );
} ib_qp_t;
Index: core/bus/kernel/bus_pnp.c
===================================================================
--- core/bus/kernel/bus_pnp.c (revision 3095)
+++ core/bus/kernel/bus_pnp.c (working copy)
@@ -53,6 +53,10 @@
#include "iba/ib_al_ifc.h"
#include "iba/ib_ci_ifc.h"
#include "iba/ib_cm_ifc.h"
+#include "al_cm_cep.h"
+#include "al_mgr.h"
+#include "bus_ev_log.h"
+#include "al_proxy.h"
/* Interface names are generated by IoRegisterDeviceInterface. */
@@ -1012,6 +1016,7 @@
p_ifc->poll_cq = ib_poll_cq;
p_ifc->rearm_cq = ib_rearm_cq;
p_ifc->join_mcast = ib_join_mcast;
+ p_ifc->join_mcast_no_qp = al_join_mcast_no_qp;
p_ifc->leave_mcast = ib_leave_mcast;
p_ifc->local_mad = ib_local_mad;
p_ifc->cm_listen = ib_cm_listen;
Index: inc/iba/ib_al.h
===================================================================
--- inc/iba/ib_al.h (revision 3095)
+++ inc/iba/ib_al.h (working copy)
@@ -3797,14 +3797,15 @@
* ib_join_mcast
*
* DESCRIPTION
-* Attaches a queue pair to a multicast group.
+* Creates a multicast group and Attaches a queue pair to it.
*
* SYNOPSIS
*/
AL_EXPORT ib_api_status_t AL_API
ib_join_mcast(
IN const ib_qp_handle_t h_qp,
- IN const ib_mcast_req_t* const p_mcast_req );
+ IN const ib_mcast_req_t* const p_mcast_req,
+ OUT ib_mcast_handle_t* p_h_mcast );
/*
* PARAMETERS
* h_qp
@@ -3872,7 +3873,85 @@
* ib_leave_mcast, ib_mcast_req_t, ib_create_qp, ib_init_dgrm_svc
*****/
+/****f* Access Layer/ib_join_mcast
+* NAME
+* ib_join_mcast_no_qp
+*
+* DESCRIPTION
+* Creates a multicast group.
+*
+* SYNOPSIS
+*/
+AL_EXPORT ib_api_status_t AL_API
+ib_join_mcast_no_qp(
+ IN const ib_mcast_req_t* const p_mcast_req,
+ OUT ib_mcast_handle_t* p_h_mcast );
+/*
+* PARAMETERS
+*
+* p_mcast_req
+* [in] Specifies the multicast group to join.
+*
+* RETURN VALUES
+* IB_SUCCESS
+* The join multicast group request has been initiated.
+*
+* IB_INVALID_QP_HANDLE
+* The queue pair handle was invalid.
+*
+* IB_INVALID_PARAMETER
+* A reference to the multicast group request information was not
+* provided.
+*
+* IB_INVALID_SERVICE_TYPE
+* The queue pair configuration does not support this type of service.
+*
+* IB_INSUFFICIENT_MEMORY
+* There was insufficient memory to join the multicast group.
+*
+* IB_INVALID_GUID
+* No port was found for the port_guid specified in the request.
+*
+* IB_INSUFFICIENT_RESOURCES
+* There were insufficient resources currently available on the channel
+* adapter to perform the operation.
+*
+* IB_INVALID_PKEY
+* The pkey specified in the multicast join request does not match the
+* pkey of the queue pair.
+*
+* IB_INVALID_PORT
+* The port GUID specified in the multicast join request does not match
+* the port of the queue pair.
+*
+* IB_ERROR
+* An error occurred while performing the multicast group join operation.
+*
+* IB_INSUFFICIENT_RESOURCES
+* There were insufficient resources currently available to complete
+* the request.
+*
+* IB_INSUFFICIENT_MEMORY
+* There was insufficient memory to complete the request.
+*
+* NOTES
+* This routine results in the specified queue pair joining a multicast
+* group. If the multicast group does not already exist, it will be created
+* at the user's option. Information about the multicast group is returned
+* to the user through a callback specified through the p_mcast_req
+* parameter.
+*
+* If the specified queue pair is already a member of a multicast group when
+* this call is invoked, an error will occur if there are conflicting
+* membership requirements. The QP is restricted to being bound to a single
+* port_guid and using a single pkey.
+*
+* SEE ALSO
+* ib_leave_mcast, ib_mcast_req_t, ib_create_qp, ib_init_dgrm_svc
+*****/
+
+
/****f* Access Layer/ib_leave_mcast
* NAME
* ib_leave_mcast
Index: inc/kernel/iba/ib_al_ifc.h
===================================================================
--- inc/kernel/iba/ib_al_ifc.h (revision 3095)
+++ inc/kernel/iba/ib_al_ifc.h (working copy)
@@ -48,7 +48,7 @@
* IB resources provided by HCAs.
*********/
-#define AL_INTERFACE_VERSION (13)
+#define AL_INTERFACE_VERSION (15)
@@ -401,9 +401,15 @@
typedef ib_api_status_t
(*ib_pfn_join_mcast_t)(
IN const ib_qp_handle_t h_qp,
- IN const ib_mcast_req_t* const p_mcast_req );
+ IN const ib_mcast_req_t* const p_mcast_req,
+ OUT ib_mcast_handle_t* p_h_mcast );
typedef ib_api_status_t
+(*ib_pfn_join_mcast_no_qp_t)(
+ IN const ib_mcast_req_t* const p_mcast_req,
+ OUT ib_mcast_handle_t* p_h_mcast );
+
+typedef ib_api_status_t
(*ib_pfn_leave_mcast_t)(
IN const ib_mcast_handle_t h_mcast,
IN const ib_pfn_destroy_cb_t destroy_cb );
@@ -702,6 +708,7 @@
ib_pfn_poll_cq_t poll_cq;
ib_pfn_rearm_cq_t rearm_cq;
ib_pfn_join_mcast_t join_mcast;
+ ib_pfn_join_mcast_no_qp_t join_mcast_no_qp;
ib_pfn_leave_mcast_t leave_mcast;
ib_pfn_local_mad_t local_mad;
ib_pfn_cm_listen_t cm_listen;
Index: ulp/ipoib/kernel/ipoib_port.c
===================================================================
--- ulp/ipoib/kernel/ipoib_port.c (revision 3095)
+++ ulp/ipoib/kernel/ipoib_port.c (working copy)
@@ -5634,7 +5634,7 @@
ipoib_port_ref( p_port, ref_join_bcast );
status = p_port->p_adapter->p_ifc->join_mcast(
- p_port->ib_mgr.h_qp, &mcast_req );
+ p_port->ib_mgr.h_qp, &mcast_req, NULL );
if( status != IB_SUCCESS )
{
ipoib_port_deref( p_port, ref_bcast_join_failed );
@@ -5695,7 +5695,7 @@
/* reference the object for the multicast join request. */
ipoib_port_ref( p_port, ref_join_bcast );
- status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp, &mcast_req );
+ status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp, &mcast_req, NULL );
if( status != IB_SUCCESS )
{
ipoib_port_deref( p_port, ref_bcast_create_failed );
@@ -6155,7 +6155,7 @@
/* reference the object for the multicast join request. */
ipoib_port_ref( p_port, ref_join_mcast );
- status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp, &mcast_req );
+ status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp, &mcast_req, NULL );
if( status != IB_SUCCESS )
{
ipoib_port_deref( p_port, ref_mcast_join_failed );
Index: ulp/ipoib_NDIS6_CM/kernel/ipoib_port.cpp
===================================================================
--- ulp/ipoib_NDIS6_CM/kernel/ipoib_port.cpp (revision 3095)
+++ ulp/ipoib_NDIS6_CM/kernel/ipoib_port.cpp (working copy)
@@ -8327,7 +8327,7 @@
/* reference the object for the multicast join request. */
ipoib_port_ref( p_port, ref_join_bcast );
- status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp, &mcast_req );
+ status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp, &mcast_req, NULL );
if( status != IB_SUCCESS )
{
ipoib_port_deref( p_port, ref_bcast_join_failed );
@@ -8388,7 +8388,7 @@
/* reference the object for the multicast join request. */
ipoib_port_ref( p_port, ref_join_bcast );
- status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp, &mcast_req );
+ status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp, &mcast_req, NULL );
if( status != IB_SUCCESS )
{
ipoib_port_deref( p_port, ref_bcast_create_failed );
@@ -8908,7 +8908,7 @@
/* reference the object for the multicast join request. */
ipoib_port_ref( p_port, ref_join_mcast );
- status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp, &mcast_req );
+ status = p_port->p_adapter->p_ifc->join_mcast( p_port->ib_mgr.h_qp, &mcast_req, NULL );
if( status != IB_SUCCESS )
{
ipoib_port_deref( p_port, ref_mcast_join_failed );
> -----Original Message-----
> From: ofw-bounces at lists.openfabrics.org [mailto:ofw-
> bounces at lists.openfabrics.org] On Behalf Of Tzachi Dar
> Sent: Tuesday, February 15, 2011 11:45 PM
> To: Hefty, Sean; ofw at lists.openfabrics.org
> Subject: Re: [ofw] patch: [ibbus] Add A new function to IBAL that allows one
> to create a multicast group without attaching a QP to it.
>
> Hi Sean,
>
> I now see why you need the h_mcast as an output parameter. We will add this to
> the function, but I don't think we will add the cancel multicast on the first
> stage.
>
> We will also see how we can track the user objects from the kernel.
>
> Thanks
> Tzachi
>
> ________________________________________
> From: Hefty, Sean [sean.hefty at intel.com]
> Sent: Tuesday, February 15, 2011 8:07 PM
> To: Tzachi Dar; ofw at lists.openfabrics.org
> Subject: RE: [ofw] patch: [ibbus] Add A new function to IBAL that allows one
> to create a multicast group without attaching a QP to it.
>
> > 1) On ib_join_mcast( ..., ib_mcast_handle_t *h_mcast); the h_mcast gets
> > back to the user from the call back.
> > I guess that there is no problem to return the h_mcast in the call itself
> > but are we sure this has a good reason?
>
> This allows the user to cancel the join operation without destroying
> everything by closing their al instance. For example, after issuing a join,
> they can immediately respond to an SM change, reregister, or other event by
> freeing the join and issuing a new one. Without this, handling those events
> becomes more difficult.
>
> Hmm... on the Linux side, the multicast module reports those types of events
> against the join request. The user can receive multiple (serialized)
> callbacks for a single join call. How does ibal handle errors on a multicast
> group after a successful join?
>
>
> > 2) " Personally, I'm fine requiring the caller to handle the cleanup
> > (provided the kernel cleans up after user space)."
> > Doesn't this has an influence on the API which would actually force us to
> > have something else in the API (for example h_al) or something similar.
>
> Not necessarily. The user to kernel proxy code needs to associate the h_mcast
> with another object, such as the file, but that doesn't have to be in the
> kernel API. The proxy code must free the multicast object when the file is
> closed. You can look at the winverbs kernel cleanup code as an example.
>
> - Sean
> _______________________________________________
> ofw mailing list
> ofw at lists.openfabrics.org
> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ofw
-------------- next part --------------
A non-text attachment was scrubbed...
Name: mcast.diff
Type: application/octet-stream
Size: 14534 bytes
Desc: mcast.diff
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20110217/5e7836e3/attachment.obj>
More information about the ofw
mailing list