[ofa-general] RFC: SRC API

[Gleb Natapov]

On Sun, Jul 29, 2007 at 05:04:31PM +0300, Michael S. Tsirkin wrote:
> Hello!
> Here is an API proposal for support of the SRC
> (scalable reliable connected) protocol extension in libibverbs.
> 
> This adds APIs to:
> - manage SRC domains
> 
> - share SRC domains between processes,
>   by means of creating a 1:1 association
>   between an SRC domain and a file.
> 
> Notes:
> - The file is specified by means of a file descriptor,
>   this makes it possible for the user to manage file
>   creation/deletion in the most flexible manner
>   (e.g. tmpfile can be used).
> 
> - I envision implementing this sharing mechanism in kernel by means
>   of a per-device tree, with inode as a key and domain object
>   as a value.
>  
> Please comment.
Can you provide a pseudo code of an application using this API?
Especially QP sharing part.

> 
> Signed-off-by: Michael S. Tsirkin <mst at dev.mellanox.co.il>
> 
> ---
> 
> diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
> index acc1b82..503f201 100644
> --- a/include/infiniband/verbs.h
> +++ b/include/infiniband/verbs.h
> @@ -370,6 +370,11 @@ struct ibv_ah_attr {
>  	uint8_t			port_num;
>  };
>  
> +struct ibv_src_domain {
> +	struct ibv_context     *context;
> +	uint32_t		handle;
> +};
> +
>  enum ibv_srq_attr_mask {
>  	IBV_SRQ_MAX_WR	= 1 << 0,
>  	IBV_SRQ_LIMIT	= 1 << 1
> @@ -389,7 +394,8 @@ struct ibv_srq_init_attr {
>  enum ibv_qp_type {
>  	IBV_QPT_RC = 2,
>  	IBV_QPT_UC,
> -	IBV_QPT_UD
> +	IBV_QPT_UD,
> +	IBV_QPT_SRC
>  };
>  
>  struct ibv_qp_cap {
> @@ -408,6 +414,7 @@ struct ibv_qp_init_attr {
>  	struct ibv_qp_cap	cap;
>  	enum ibv_qp_type	qp_type;
>  	int			sq_sig_all;
> +	struct ibv_src_domain  *src_domain;
>  };
>  
>  enum ibv_qp_attr_mask {
> @@ -526,6 +533,7 @@ struct ibv_send_wr {
>  			uint32_t	remote_qkey;
>  		} ud;
>  	} wr;
> +	uint32_t		src_remote_srq_num;
>  };
>  
>  struct ibv_recv_wr {
> @@ -553,6 +561,10 @@ struct ibv_srq {
>  	pthread_mutex_t		mutex;
>  	pthread_cond_t		cond;
>  	uint32_t		events_completed;
> +
> +	uint32_t		src_srq_num;
> +	struct ibv_src_domain  *src_domain;
> +	struct ibv_cq	       *src_cq;
>  };
>  
>  struct ibv_qp {
> @@ -570,6 +582,8 @@ struct ibv_qp {
>  	pthread_mutex_t		mutex;
>  	pthread_cond_t		cond;
>  	uint32_t		events_completed;
> +
> +	struct ibv_src_domain  *src_domain;
>  };
>  
>  struct ibv_comp_channel {
> @@ -912,6 +926,25 @@ struct ibv_srq *ibv_create_srq(struct ibv_pd *pd,
>  			       struct ibv_srq_init_attr *srq_init_attr);
>  
>  /**
> + * ibv_create_src_srq - Creates a SRQ associated with the specified protection
> + *   domain and src domain.
> + * @pd: The protection domain associated with the SRQ.
> + * @src_domain: The SRC domain associated with the SRQ.
> + * @src_cq: CQ to report completions for SRC packets on.
> + *
> + * @srq_init_attr: A list of initial attributes required to create the SRQ.
> + *
> + * srq_attr->max_wr and srq_attr->max_sge are read the determine the
> + * requested size of the SRQ, and set to the actual values allocated
> + * on return.  If ibv_create_srq() succeeds, then max_wr and max_sge
> + * will always be at least as large as the requested values.
> + */
> +struct ibv_srq *ibv_create_src_srq(struct ibv_pd *pd,
> +				   struct ibv_src_domain *src_domain,
> +				   struct ibv_cq *src_cq,
> +			           struct ibv_srq_init_attr *srq_init_attr);
> +
> +/**
>   * ibv_modify_srq - Modifies the attributes for the specified SRQ.
>   * @srq: The SRQ to modify.
>   * @srq_attr: On input, specifies the SRQ attributes to modify.  On output,
> @@ -1074,6 +1107,44 @@ int ibv_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid);
>   */
>  int ibv_fork_init(void);
>  
> +/**
> + * ibv_alloc_src_domain - Allocate an SRC domain
> + * Returns a reference to an SRC domain.
> + * Use ibv_put_src_domain to free the reference.
> + * @context: Device context
> + */
> +struct ibv_src_domain *ibv_get_new_src_domain(struct ibv_context *context);
> +
> +/**
> + * ibv_share_src_domain - associate the src domain with a file.
> + * Establishes a connection between an SRC domain object and a file descriptor.
> + *
> + * @d: SRC domain to share
> + * @fd: descriptor for a file to associate with the domain
> + */
> +int ibv_share_src_domain(struct ibv_src_domain *d, int fd);
> +
> +/**
> + * ibv_unshare_src_domain - disassociate the src domain from a file.
> + * Subsequent calls to ibv_get_shared_src_domain will fail.
> + * @d: SRC domain to unshare
> + */
> +int ibv_unshare_src_domain(struct ibv_src_domain *d);
> +
> +/**
> + * ibv_get_src_domain - get a reference to shared SRC domain
> + * @context: Device context
> + * @fd: descriptor for a file associated with the domain
> + */
> +struct ibv_src_domain *ibv_get_shared_src_domain(struct ibv_context *context, int fd);
> +
> +/**
> + * ibv_put_src_domain - destroy a reference to an SRC domain
> + * If this is the last reference, destroys the domain.
> + * @d: reference to SRC domain to put
> + */
> +int ibv_put_src_domain(struct ibv_src_domain *d);
> +
>  END_C_DECLS
>  
>  #  undef __attribute_const
> 
> 
> 
> -- 
> MST
> _______________________________________________
> general mailing list
> general at lists.openfabrics.org
> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
> 
> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general

--
			Gleb.