[openib-general] Some ib_mad.h Redirection Comments
Sean Hefty
mshefty at ichips.intel.com
Mon Aug 9 18:36:14 PDT 2004
On Sat, 7 Aug 2004 22:29:18 +0300
"Yaron Haviv" <yaronh at voltaire.com> wrote:
> > If we're in agreement on the server side, then I think we're more than
> > half-way there.
>
> To me it looks like there is agreement :)
I've updated ib_mad.h (again). This time additional comments were added, and notes were made where we need to continue or start having discussions. The only major change to the API was modifying ib_mad_recv_wc to use a single receive buffer, versus a chain. (I haven't given up on zero-copy receives; the proposed chaining just needs some additional thought.) Hopefully, we can begin working towards this API, but continue discussing some of the areas marked in the file with 'XXX'.
> So that's why I think Redirect in CM & SA are not the same and may
> require different approach
>
> Your thoughts ?
I need to think more about the redirection case, and examine an implementation to see how much work/policy is in it. I think that if it is totally transparent to the requestor, then the API will be unaffected. Likewise, if it is exposed. We probably only need to modify the API if we need to reach some sort of compromise between the implementation versus the policy.
Modified file is below:
/*
This software is available to you under a choice of one of two
licenses. You may choose to be licensed under the terms of the GNU
General Public License (GPL) Version 2, available at
<http://www.fsf.org/copyleft/gpl.html>, or the OpenIB.org BSD
license, available in the LICENSE.TXT file accompanying this
software. These details are also available at
<http://openib.org/license.html>.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
Copyright (c) 2004 Infinicon Corporation. All rights reserved.
Copyright (c) 2004 Intel Corporation. All rights reserved.
Copyright (c) 2004 Topspin Corporation. All rights reserved.
Copyright (c) 2004 Voltaire Corporation. All rights reserved.
*/
#if !defined( IB_MAD_H )
#define IB_MAD_H
#include "ib_verbs.h"
struct ib_grh {
u32 version_tclass_flow;
u16 paylen;
u8 next_hdr;
u8 hop_limit;
union ib_gid sgid;
union ib_gid dgid;
} __attribute__ ((packed));
struct ib_mad_hdr {
u8 base_version;
u8 mgmt_class;
u8 class_version;
u8 method;
u16 status;
u16 class_specific;
u64 tid;
u16 attr_id;
u16 resv;
u32 attr_mod;
} __attribute__ ((packed));
struct ib_rmpp_hdr {
u8 rmpp_version;
u8 rmpp_type;
u8 rmpp_flags;
u8 rmpp_status;
u32 seg_num;
u32 paylen_newwin;
} __attribute__ ((packed));
struct ib_mad {
struct ib_mad_hdr mad_hdr;
u8 data[232];
} __attribute__ ((packed));
struct ib_rmpp_mad {
struct ib_mad_hdr mad_hdr;
struct ib_rmpp_hdr rmpp_hdr;
u8 data[220];
} __attribute__ ((packed));
struct ib_mad_agent;
struct ib_mad_send_wc;
struct ib_mad_recv_wc;
/**
* ib_mad_send_handler - callback handler for a sent MAD.
* @mad_agent - MAD agent that sent the MAD.
* @mad_send_wc - Send work completion information on the sent MAD.
*/
typedef void (*ib_mad_send_handler)(struct ib_mad_agent *mad_agent,
struct ib_mad_send_wc *mad_send_wc);
/**
* ib_mad_recv_handler - callback handler for a received MAD.
* @mad_agent - MAD agent requesting the received MAD.
* @mad_recv_wc - Received work completion information on the received MAD.
*
* MADs received in response to a send request operation will be handed to
* the user after the send operation completes. All data buffers given
* to the user through this routine are owned by the receiving client.
*/
typedef void (*ib_mad_recv_handler)(struct ib_mad_agent *mad_agent,
struct ib_mad_recv_wc *mad_recv_wc);
/**
* ib_mad_agent - Used to track MAD registration with the access layer.
* @device - Reference to device registration is on.
* @qp - Reference to QP used for sending and receiving MADs.
* @recv_handler - Callback handler for a received MAD.
* @send_handler - Callback hander for a sent MAD.
* @context - User-specified context associated with this registration.
* @hi_tid - Access layer assigned transition ID for this client.
* Unsolicited MADs sent by this client will have the upper 32-bits
* of their TID set to this value.
*/
struct ib_mad_agent {
struct ib_device *device;
struct ib_qp *qp;
ib_mad_recv_handler recv_handler;
ib_mad_send_handler send_handler;
void *context;
u32 hi_tid;
};
enum ib_mad_flags {
IB_MAD_GRH_VALID = 1
};
/**
* ib_mad_send_wr - send MAD work request.
* @list - Allows chaining together multiple requests.
* @context - User-controlled work request context.
* @sg_list - An array of scatter-gather entries, referencing the MAD's
* data buffer(s). The first entry must reference the standard MAD
* header, plus any RMPP header, if used.
* @num_sge - The number of scatter-gather entries.
* @mad_flags - Flags used to control the send operation.
* @ah - Address handle for the destination.
* @timeout_ms - Timeout value, in milliseconds, to wait for a response
* message. Set to 0 if no response is expected.
* @remote_qpn - Destination QP.
* @remote_qkey - Specifies the qkey used by remote QP.
* @pkey_index - Pkey index to use. Required when sending on QP1 only.
*/
/* XXX See about using ib_send_wr directly, e.g.:
context -> wr_id
mad_flags -> send_flags
add new timeout_ms field or double use of imm_data
*/
struct ib_mad_send_wr {
struct list_head list;
void *context;
struct ib_sge *sg_list;
int num_sge;
int mad_flags;
struct ib_ah *ah;
int timeout_ms;
u32 remote_qpn;
u32 remote_qkey;
u16 pkey_index;
};
/**
* ib_mad_send_wc - MAD send completion information.
* @context - Context associated with the send MAD request.
* @status - Completion status.
* @vendor_err - Optional vendor error information returned with a failed
* request.
*/
struct ib_mad_send_wc {
void *context;
enum ib_wc_status status;
u32 vendor_err;
};
/**
* ib_mad_recv_wc - received MAD information.
* @context - For received response, set to the context specified for
* the corresponding send request.
* @grh - References a data buffer containing the global route header.
* The data refereced by this buffer is only valid if the GRH is
* valid.
* @mad - References the start of the received MAD.
* @length - Specifies the size of the received MAD.
* @mad_flags - Flags used to specify information about the received MAD.
* @mad_len - The length of the received MAD, without duplicated headers.
* @src_qpn - Source QP.
* @pkey_index - Pkey index.
* @slid - LID of remote QP.
* @sl - Service level of source for a received message.
* @dlid_path_bits - Path bits of source for a received message.
*
* An RMPP receive will be coalesced into a single data buffer.
*/
/* XXX revisit possibility of zero-copy receive */
struct ib_mad_recv_wc {
void *context;
struct ib_grh *grh;
struct ib_mad *mad;
u32 length;
int mad_flags;
u32 mad_len;
u32 src_qp;
u16 pkey_index;
u16 slid;
u8 sl;
u8 dlid_path_bits;
};
/**
* ib_mad_reg_req - MAD registration request
* @mgmt_class - Indicates which management class of MADs should be receive
* by the caller. This field is only required if the user wishes to
* receive unsolicited MADs, otherwise it should be 0.
* @mgmt_class_version - Indicates which version of MADs for the given
* management class to receive.
* @method_mask - The caller will receive unsolicited MADs for any method
* where @method_mask = 1.
*/
/* XXX Need to extend to support snooping - perhaps registration type
with masks for the class, version, methods if type is 'view-only'?
*/
struct ib_mad_reg_req {
u8 mgmt_class;
u8 mgmt_class_version;
DECLARE_BITMAP(method_mask, 128);
};
/**
* ib_mad_reg - Register to send/receive MADs.
* @device - The device to register with.
* @port - The port on the specified device to use.
* @qp_type - Specifies which QP to access. Must be either
* IB_QPT_SMI or IB_QPT_GSI.
* @mad_reg_req - Specifies which unsolicited MADs should be received
* by the caller. This parameter may be NULL if the caller only
* wishes to receive solicited responses.
* @rmpp_version - If set, indicates that the client will send
* and receive MADs that contain the RMPP header for the given version.
* If set to 0, indicates that RMPP is not used by this client.
* @send_handler - The completion callback routine invoked after a send
* request has completed.
* @recv_handler - The completion callback routine invoked for a received
* MAD.
* @context - User specified context associated with the registration.
*/
struct ib_mad_agent *ib_mad_reg(struct ib_device *device,
u8 port,
enum ib_qp_type qp_type,
struct ib_mad_reg_req *mad_reg_req,
u8 rmpp_version,
ib_mad_send_handler send_handler,
ib_mad_recv_handler recv_handler,
void *context);
/**
* ib_mad_dereg - Deregisters a client from using MAD services.
* @mad_agent - Corresponding MAD registration request to deregister.
*
* After invoking this routine, MAD services are no longer usable by the
* client on the associated QP.
*/
int ib_mad_dereg(struct ib_mad_agent *mad_agent);
/**
* ib_mad_post_send - Posts a MAD to the send queue of the QP associated
* with the registered client.
* @mad_agent - Specifies the associated registration to post the send to.
* @mad_send_wr - Specifies the information needed to send the MAD.
*/
/* XXX Need to define queuing model - above or below API? */
int ib_mad_post_send(struct ib_mad_agent *mad_agent,
struct ib_mad_send_wr *mad_send_wr);
/**
* ib_mad_qp_redir - Registers a QP for MAD services.
* @qp - Reference to a QP that requires MAD services.
* @rmpp_version - If set, indicates that the client will send
* and receive MADs that contain the RMPP header for the given version.
* If set to 0, indicates that RMPP is not used by this client.
* @send_handler - The completion callback routine invoked after a send
* request has completed.
* @recv_handler - The completion callback routine invoked for a received
* MAD.
* @context - User specified context associated with the registration.
*
* Use of this call allows clients to use MAD services, such as RMPP,
* on user-owned QPs. After calling this routine, users may send
* MADs on the specified QP by calling ib_mad_post_send.
*/
/* XXX Need to define provided features for requestor-side redirecting */
struct ib_mad_agent *ib_mad_qp_redir(struct ib_qp *qp,
u8 rmpp_version,
ib_mad_send_handler send_handler,
ib_mad_recv_handler recv_handler,
void *context);
/**
* ib_mad_process_wc - Processes a work completion associated with a
* MAD sent or received on a redirected QP.
* @mad_agent - Specifies the registered MAD service using the redirected QP.
* @wc - References a work completion associated with a sent or received
* MAD segment.
*
* This routine is used to complete or continue processing on a MAD request.
* If the work completion is associated with a send operation, calling
* this routine is required to continue an RMPP transfer or to wait for a
* corresponding response, if it is a request. If the work completion is
* associated with a receive operation, calling this routine is required to
* process an inbound or outbound RMPP transfer, or to match a response MAD
* with its corresponding request.
*/
int ib_mad_process_wc(struct ib_mad_agent *mad_agent,
struct ib_wc *wc);
#endif /* IB_MAD_H */
More information about the general
mailing list