[ofa-general] RFC: SRC API

[Michael S. Tsirkin]

Hello!
Here is an API proposal for support of the SRC
(scalable reliable connected) protocol extension in libibverbs.

This adds APIs to:
- manage SRC domains

- share SRC domains between processes,
  by means of creating a 1:1 association
  between an SRC domain and a file.

Notes:
- The file is specified by means of a file descriptor,
  this makes it possible for the user to manage file
  creation/deletion in the most flexible manner
  (e.g. tmpfile can be used).

- I envision implementing this sharing mechanism in kernel by means
  of a per-device tree, with inode as a key and domain object
  as a value.
 
Please comment.

Signed-off-by: Michael S. Tsirkin <mst at dev.mellanox.co.il>

---

diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index acc1b82..503f201 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -370,6 +370,11 @@ struct ibv_ah_attr {
 	uint8_t			port_num;
 };
 
+struct ibv_src_domain {
+	struct ibv_context     *context;
+	uint32_t		handle;
+};
+
 enum ibv_srq_attr_mask {
 	IBV_SRQ_MAX_WR	= 1 << 0,
 	IBV_SRQ_LIMIT	= 1 << 1
@@ -389,7 +394,8 @@ struct ibv_srq_init_attr {
 enum ibv_qp_type {
 	IBV_QPT_RC = 2,
 	IBV_QPT_UC,
-	IBV_QPT_UD
+	IBV_QPT_UD,
+	IBV_QPT_SRC
 };
 
 struct ibv_qp_cap {
@@ -408,6 +414,7 @@ struct ibv_qp_init_attr {
 	struct ibv_qp_cap	cap;
 	enum ibv_qp_type	qp_type;
 	int			sq_sig_all;
+	struct ibv_src_domain  *src_domain;
 };
 
 enum ibv_qp_attr_mask {
@@ -526,6 +533,7 @@ struct ibv_send_wr {
 			uint32_t	remote_qkey;
 		} ud;
 	} wr;
+	uint32_t		src_remote_srq_num;
 };
 
 struct ibv_recv_wr {
@@ -553,6 +561,10 @@ struct ibv_srq {
 	pthread_mutex_t		mutex;
 	pthread_cond_t		cond;
 	uint32_t		events_completed;
+
+	uint32_t		src_srq_num;
+	struct ibv_src_domain  *src_domain;
+	struct ibv_cq	       *src_cq;
 };
 
 struct ibv_qp {
@@ -570,6 +582,8 @@ struct ibv_qp {
 	pthread_mutex_t		mutex;
 	pthread_cond_t		cond;
 	uint32_t		events_completed;
+
+	struct ibv_src_domain  *src_domain;
 };
 
 struct ibv_comp_channel {
@@ -912,6 +926,25 @@ struct ibv_srq *ibv_create_srq(struct ibv_pd *pd,
 			       struct ibv_srq_init_attr *srq_init_attr);
 
 /**
+ * ibv_create_src_srq - Creates a SRQ associated with the specified protection
+ *   domain and src domain.
+ * @pd: The protection domain associated with the SRQ.
+ * @src_domain: The SRC domain associated with the SRQ.
+ * @src_cq: CQ to report completions for SRC packets on.
+ *
+ * @srq_init_attr: A list of initial attributes required to create the SRQ.
+ *
+ * srq_attr->max_wr and srq_attr->max_sge are read the determine the
+ * requested size of the SRQ, and set to the actual values allocated
+ * on return.  If ibv_create_srq() succeeds, then max_wr and max_sge
+ * will always be at least as large as the requested values.
+ */
+struct ibv_srq *ibv_create_src_srq(struct ibv_pd *pd,
+				   struct ibv_src_domain *src_domain,
+				   struct ibv_cq *src_cq,
+			           struct ibv_srq_init_attr *srq_init_attr);
+
+/**
  * ibv_modify_srq - Modifies the attributes for the specified SRQ.
  * @srq: The SRQ to modify.
  * @srq_attr: On input, specifies the SRQ attributes to modify.  On output,
@@ -1074,6 +1107,44 @@ int ibv_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid);
  */
 int ibv_fork_init(void);
 
+/**
+ * ibv_alloc_src_domain - Allocate an SRC domain
+ * Returns a reference to an SRC domain.
+ * Use ibv_put_src_domain to free the reference.
+ * @context: Device context
+ */
+struct ibv_src_domain *ibv_get_new_src_domain(struct ibv_context *context);
+
+/**
+ * ibv_share_src_domain - associate the src domain with a file.
+ * Establishes a connection between an SRC domain object and a file descriptor.
+ *
+ * @d: SRC domain to share
+ * @fd: descriptor for a file to associate with the domain
+ */
+int ibv_share_src_domain(struct ibv_src_domain *d, int fd);
+
+/**
+ * ibv_unshare_src_domain - disassociate the src domain from a file.
+ * Subsequent calls to ibv_get_shared_src_domain will fail.
+ * @d: SRC domain to unshare
+ */
+int ibv_unshare_src_domain(struct ibv_src_domain *d);
+
+/**
+ * ibv_get_src_domain - get a reference to shared SRC domain
+ * @context: Device context
+ * @fd: descriptor for a file associated with the domain
+ */
+struct ibv_src_domain *ibv_get_shared_src_domain(struct ibv_context *context, int fd);
+
+/**
+ * ibv_put_src_domain - destroy a reference to an SRC domain
+ * If this is the last reference, destroys the domain.
+ * @d: reference to SRC domain to put
+ */
+int ibv_put_src_domain(struct ibv_src_domain *d);
+
 END_C_DECLS
 
 #  undef __attribute_const



-- 
MST