[openib-general] [PATCH] new libibverbs handling of stale CQ events
Roland Dreier
rolandd at cisco.com
Tue Sep 6 14:30:09 PDT 2005
Again, completely analogous to the existing stale async event handling...
- R.
--- libibverbs/include/infiniband/verbs.h (revision 3324)
+++ libibverbs/include/infiniband/verbs.h (working copy)
@@ -502,7 +502,8 @@ struct ibv_cq {
pthread_mutex_t mutex;
pthread_cond_t cond;
- uint32_t events_completed;
+ uint32_t comp_events_completed;
+ uint32_t async_events_completed;
};
struct ibv_ah {
@@ -608,21 +609,22 @@ extern int ibv_close_device(struct ibv_c
* ibv_get_async_event - Get next async event
* @event: Pointer to use to return async event
*
- * The event returned must eventually be released via ibv_put_async_event().
+ * All async events returned by ibv_get_async_event() must eventually
+ * be acknowledged with ibv_ack_async_event().
*/
extern int ibv_get_async_event(struct ibv_context *context,
struct ibv_async_event *event);
/**
- * ibv_put_async_event - Free an async event
- * @event: Event to be released.
+ * ibv_ack_async_event - Free an async event
+ * @event: Event to be acknowledged.
*
- * All events which are returned by ib_get_async_event() must be
- * released. There should be a one-to-one correspondence between
- * successful gets and puts.
+ * All async events which are returned by ibv_get_async_event() must
+ * be acknowledged. Destroying an object (CQ, SRQ or QP) will wait
+ * for all affiliated events to be acknowledged, so there should be a
+ * one-to-one correspondence between acks and successful gets.
*/
-extern void ibv_put_async_event(struct ibv_async_event *event);
-
+extern void ibv_ack_async_event(struct ibv_async_event *event);
/**
* ibv_query_device - Get device properties
@@ -682,10 +684,31 @@ extern int ibv_destroy_cq(struct ibv_cq
/**
* ibv_get_cq_event - Read next CQ event
+ * @context: Context to get CQ event for
+ * @comp_num: Index of completion event to check. Must be >= 0 and
+ * <= context->num_comp.
+ * @cq: Used to return pointer to CQ.
+ * @cq_context: Used to return consumer-supplied CQ context.
+ *
+ * All completion events returned by ibv_get_cq_event() must
+ * eventually be acknowledged with ibv_ack_cq_events().
*/
extern int ibv_get_cq_event(struct ibv_context *context, int comp_num,
struct ibv_cq **cq, void **cq_context);
-
+
+/**
+ * ibv_ack_cq_events - Free an async event
+ * @cq: CQ to acknowledge events for
+ * @nevents: Number of events to acknowledge.
+ *
+ * All completion events which are returned by ibv_get_cq_event() must
+ * be acknowledged. ibv_destroy_cq() will wait for all completion
+ * events to be acknowledged, so there should be a one-to-one
+ * correspondence between acks and successful gets. An application
+ * may accumulate multiple completion events and acknowledge them in a
+ * single call by passing the number of events to ack in @nevents.
+ */
+extern void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents);
/**
* ibv_poll_cq - Poll a CQ for work completions
--- libibverbs/include/infiniband/kern-abi.h (revision 3324)
+++ libibverbs/include/infiniband/kern-abi.h (working copy)
@@ -335,7 +335,8 @@ struct ibv_destroy_cq {
};
struct ibv_destroy_cq_resp {
- __u32 events_reported;
+ __u32 comp_events_reported;
+ __u32 async_events_reported;
};
struct ibv_create_qp {
--- libibverbs/src/libibverbs.map (revision 3324)
+++ libibverbs/src/libibverbs.map (working copy)
@@ -6,7 +6,7 @@ IBVERBS_1.0 {
ibv_open_device;
ibv_close_device;
ibv_get_async_event;
- ibv_put_async_event;
+ ibv_ack_async_event;
ibv_query_device;
ibv_query_port;
ibv_query_gid;
@@ -18,6 +18,7 @@ IBVERBS_1.0 {
ibv_create_cq;
ibv_destroy_cq;
ibv_get_cq_event;
+ ibv_ack_cq_events;
ibv_create_srq;
ibv_modify_srq;
ibv_destroy_srq;
--- libibverbs/src/device.c (revision 3324)
+++ libibverbs/src/device.c (working copy)
@@ -171,7 +171,7 @@ int ibv_get_async_event(struct ibv_conte
return 0;
}
-void ibv_put_async_event(struct ibv_async_event *event)
+void ibv_ack_async_event(struct ibv_async_event *event)
{
switch (event->event_type) {
case IBV_EVENT_CQ_ERR:
@@ -179,7 +179,7 @@ void ibv_put_async_event(struct ibv_asyn
struct ibv_cq *cq = event->element.cq;
pthread_mutex_lock(&cq->mutex);
- ++cq->events_completed;
+ ++cq->async_events_completed;
pthread_cond_signal(&cq->cond);
pthread_mutex_unlock(&cq->mutex);
--- libibverbs/src/verbs.c (revision 3324)
+++ libibverbs/src/verbs.c (working copy)
@@ -107,9 +107,10 @@ struct ibv_cq *ibv_create_cq(struct ibv_
struct ibv_cq *cq = context->ops.create_cq(context, cqe);
if (cq) {
- cq->context = context;
- cq->cq_context = cq_context;
- cq->events_completed = 0;
+ cq->context = context;
+ cq->cq_context = cq_context;
+ cq->comp_events_completed = 0;
+ cq->async_events_completed = 0;
pthread_mutex_init(&cq->mutex, NULL);
pthread_cond_init(&cq->cond, NULL);
}
@@ -143,6 +144,14 @@ int ibv_get_cq_event(struct ibv_context
return 0;
}
+void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
+{
+ pthread_mutex_lock(&cq->mutex);
+ cq->comp_events_completed += nevents;
+ pthread_cond_signal(&cq->cond);
+ pthread_mutex_unlock(&cq->mutex);
+}
+
struct ibv_srq *ibv_create_srq(struct ibv_pd *pd,
struct ibv_srq_init_attr *srq_init_attr)
{
--- libibverbs/src/cmd.c (revision 3324)
+++ libibverbs/src/cmd.c (working copy)
@@ -303,7 +303,8 @@ int ibv_cmd_destroy_cq(struct ibv_cq *cq
return errno;
pthread_mutex_lock(&cq->mutex);
- while (cq->events_completed != resp.events_reported)
+ while (cq->comp_events_completed != resp.comp_events_reported ||
+ cq->async_events_completed != resp.async_events_reported)
pthread_cond_wait(&cq->cond, &cq->mutex);
pthread_mutex_unlock(&cq->mutex);
--- libibverbs/ChangeLog (revision 3324)
+++ libibverbs/ChangeLog (working copy)
@@ -1,8 +1,16 @@
+2005-09-06 Roland Dreier <roland at cisco.com>
+
+ * include/infiniband/kern-abi.h, include/infiniband/verbs.h,
+ src/cmd.c, src/device.c, src/verbs.c, examples/asyncwatch.c:
+ Update to handle new kernel ABI for avoiding stale completion
+ events. This is completely analogous to the previous asynchronous
+ event change.
+
2005-08-31 Roland Dreier <roland at cisco.com>
* include/infiniband/kern-abi.h, include/infiniband/verbs.h,
src/cmd.c, src/device.c, src/ibverbs.h, src/init.c, src/verbs.c,
- examples/asyncwatch.h: Update to handle new kernel ABI for
+ examples/asyncwatch.c: Update to handle new kernel ABI for
avoiding stale asynchronous events. When a CQ, QP or SRQ is
destroyed, the kernel reports the number of events it has given to
userspace, and we wait until we've handled the same number of
--- libibverbs/examples/asyncwatch.c (revision 3324)
+++ libibverbs/examples/asyncwatch.c (working copy)
@@ -86,7 +86,7 @@ int main(int argc, char *argv[])
printf(" event_type %d, port %d\n", event.event_type,
event.element.port_num);
- ibv_put_async_event(&event);
+ ibv_ack_async_event(&event);
}
return 0;
More information about the general
mailing list