[openib-general] Re: [PATCH][KDAPL] clean up code : evd flow
James Lentini
jlentini at netapp.com
Mon May 9 10:01:08 PDT 2005
Itamar,
Could this patch have caused the error below?
drivers/infiniband/ulp/dat-provider/dapl_evd_connection_callb.c:100:
spin_lock(drivers/infiniband/ulp/dat-provider/dapl_ep_util.c:d740a428)
already locked by
drivers/infiniband/ulp/dat-provider/dapl_evd_connection_callb.c/100
drivers/infiniband/ulp/dat-provider/dapl_evd_connection_callb.c:152:
spin_unlock(drivers/infiniband/ulp/dat-provider/dapl_ep_util.c:d740a428)
not locked
Unable to handle kernel NULL pointer dereference at virtual address
00000048
printing eip:
e0a4715b
*pde = 00000000
Oops: 0000 [#1]
Modules linked in: kdapltest ib_dat_provider dat ib_cm ib_at ib_ipoib
ib_sa md5 ipv6 parport_pc lp parport autofs4 nfs lockd sunrpc
i2c_piix4 i2c_core ib_mthca ib_mad ib_core e100 mii floppy sg aic7xxx
sd_mod scsi_mod
CPU: 0
EIP: 0060:[<e0a4715b>] Not tainted VLI
EFLAGS: 00010096 (2.6.11-openib)
EIP is at ib_destroy_cm_id+0xb/0x790 [ib_cm]
eax: 00000000 ebx: 00000000 ecx: d92f84e4 edx: d75234a0
esi: 00000296 edi: d92f84e0 ebp: e1ac6f4f esp: dfd9becc
ds: 007b es: 007b ss: 0068
Process dapl_cm_wq/0 (pid: 5625, threadinfo=dfd9b000 task=d74268d0)
Stack: 00000000 56360d6c 0004d7ea d7426350 d74268d0 d7426350 d74268d0
de14f9c0
dfd9bf40 c035fee4 d75234e0 00000001 00000003 d75234c8 00000001
00000003
dfd9bf40 00000096 00000000 00000292 000000c2 dbd75330 000000c1
d92f84e0
Call Trace:
[<c035fee4>] schedule+0x2e4/0x620
[<e1ac6f70>] dapl_ib_destroy_cm_id_work+0x21/0x2e [ib_dat_provider]
[<c0130ac1>] worker_thread+0x211/0x440
[<c0114240>] default_wake_function+0x0/0x10
[<c0114240>] default_wake_function+0x0/0x10
[<c01308b0>] worker_thread+0x0/0x440
[<c0137366>] kthread+0x96/0xe0
[<c01372d0>] kthread+0x0/0xe0
[<c01007ed>] kernel_thread_helper+0x5/0x18
Code: 85 c0 75 02 5b c3 e8 f5 fc ff ff 8b 83 88 00 00 00 e8 ba 94 70
df 31 c0 89 83 88 00 00 00 5b c3 57 56 53 83 ec 5c 89 c3 9c 5e fa <81>
7b 48 3c 4b 24 1d 74 25 c7 04 24 d4 f3 a4 e0 8d 43 48 b9 6c
On Thu, 5 May 2005, Itamar wrote:
> Clean up kdapl : evd flow
>
> Signed-off-by: Itamar Rabenstein <itamar at mellanox.co.il>
>
> Index: dapl_evd_dto_callb.c
> ===================================================================
> --- dapl_evd_dto_callb.c (revision 2257)
> +++ dapl_evd_dto_callb.c (working copy)
> @@ -98,46 +98,26 @@ void dapl_evd_dto_callback(ib_cq_handle_
> * EVD is enabled.
> */
>
> - if (state == DAPL_EVD_STATE_WAITED) {
> - /*
> - * If we could, it would be best to avoid this wakeup
> - * (and the context switch) unless the number of events/CQs
> - * waiting for the waiter was its threshold. We don't
> - * currently have the ability to determine that without
> - * dequeueing the events, and we can't do that for
> - * synchronization reasons (racing with the waiter waking
> - * up and dequeuing, sparked by other callbacks).
> - */
> + if (state == DAPL_EVD_STATE_OPEN && (evd_ptr->cno_ptr != NULL)) {
> + /*
> + * Re-enable callback, *then* trigger.
> + * This guarantees we won't miss any events.
> + */
> + dat_status = dapl_ib_completion_notify(evd_ptr,
> + IB_NOTIFY_ON_NEXT_COMP);
>
> - /*
> - * We don't need to worry about taking the lock for the
> - * wakeup because wakeups are sticky.
> - */
> - dapl_os_wait_object_wakeup(&evd_ptr->wait_object);
> - } else if (state == DAPL_EVD_STATE_OPEN) {
> - DAPL_CNO *cno = evd_ptr->cno_ptr;
> - if (evd_ptr->evd_enabled && (evd_ptr->cno_ptr != NULL)) {
> - /*
> - * Re-enable callback, *then* trigger.
> - * This guarantees we won't miss any events.
> - */
> - dat_status = dapl_ib_completion_notify(evd_ptr,
> - IB_NOTIFY_ON_NEXT_COMP);
> -
> - if (DAT_SUCCESS != dat_status) {
> - (void)dapl_evd_post_async_error_event(evd_ptr->
> - header.
> - owner_ia->
> + if (DAT_SUCCESS != dat_status) {
> + (void)dapl_evd_post_async_error_event(evd_ptr->
> + header.
> + owner_ia->
> async_error_evd,
> DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR,
> (DAT_IA_HANDLE)
> evd_ptr->
> header.
> owner_ia);
> - }
> -
> - dapl_cno_trigger(cno, evd_ptr);
> - }
> - }
> + }
> + dapl_cno_trigger(evd_ptr->cno_ptr, evd_ptr);
> + }
> dapl_dbg_log(DAPL_DBG_TYPE_RTN, "dapl_evd_dto_callback () returns\n");
> }
> Index: dapl_ep_create.c
> ===================================================================
> --- dapl_ep_create.c (revision 2257)
> +++ dapl_ep_create.c (working copy)
> @@ -74,7 +74,6 @@ dapl_ep_create(DAT_IA_HANDLE ia_handle,
> DAPL_IA *ia_ptr;
> DAPL_EP *ep_ptr;
> DAT_EP_ATTR ep_attr_limit;
> - DAPL_EVD *evd_ptr;
> DAT_RETURN dat_status;
>
> dapl_dbg_log(DAPL_DBG_TYPE_API,
> @@ -194,39 +193,6 @@ dapl_ep_create(DAT_IA_HANDLE ia_handle,
> }
> }
>
> - /*
> - * Verify the completion flags for the EVD and the EP
> - */
> - /*
> - * XXX FIXME
> - * XXX Need to make assign the EVD to the right completion type
> - * XXX depending on the EP attributes. Fail if the types don't
> - * XXX match, they are mutually exclusive.
> - */
> - evd_ptr = (DAPL_EVD *) recv_evd_handle;
> - if (evd_ptr != NULL && evd_ptr->completion_type == DAPL_EVD_STATE_INIT) {
> - if (ep_attr != NULL &&
> - ep_attr->recv_completion_flags ==
> - DAT_COMPLETION_DEFAULT_FLAG) {
> - evd_ptr->completion_type = DAPL_EVD_STATE_THRESHOLD;
> - } else {
> - evd_ptr->completion_type =
> - ep_attr->recv_completion_flags;
> - }
> - }
> -
> - evd_ptr = (DAPL_EVD *) request_evd_handle;
> - if (evd_ptr != NULL && evd_ptr->completion_type == DAPL_EVD_STATE_INIT) {
> - if (ep_attr != NULL &&
> - ep_attr->recv_completion_flags ==
> - DAT_COMPLETION_DEFAULT_FLAG) {
> - evd_ptr->completion_type = DAPL_EVD_STATE_THRESHOLD;
> - } else {
> - evd_ptr->completion_type =
> - ep_attr->recv_completion_flags;
> - }
> - }
> -
> /* Allocate EP */
> ep_ptr = dapl_ep_alloc(ia_ptr, ep_attr);
> if (ep_ptr == NULL) {
> Index: dapl_openib_dto.h
> ===================================================================
> --- dapl_openib_dto.h (revision 2257)
> +++ dapl_openib_dto.h (working copy)
> @@ -167,5 +167,6 @@ dapl_ib_post_send(DAPL_EP * ep_ptr,
> #define DAPL_GET_CQE_OPTYPE(cqe_p) ((ib_work_completion_t *)cqe_p)->opcode
> #define DAPL_GET_CQE_BYTESNUM(cqe_p) ((ib_work_completion_t *)cqe_p)->byte_len
> #define DAPL_GET_CQE_STATUS(cqe_p) ((ib_work_completion_t *)cqe_p)->status
> +#define DAPL_GET_CQE_VND_SYND(cqe_p) ((ib_work_completion_t *)cqe_ptr)->vendor_err
>
> #endif /* DAPL_OPENIB_DTO_H */
> Index: dapl_evd_resize.c
> ===================================================================
> --- dapl_evd_resize.c (revision 2257)
> +++ dapl_evd_resize.c (working copy)
> @@ -93,13 +93,6 @@ DAT_RETURN dapl_evd_resize(DAT_EVD_HANDL
>
> spin_lock(&evd_ptr->header.lock);
>
> - /* Don't try to resize if we are actively waiting */
> - if (evd_ptr->evd_state == DAPL_EVD_STATE_WAITED) {
> - spin_unlock(&evd_ptr->header.lock);
> - dat_status = DAT_ERROR(DAT_INVALID_STATE, 0);
> - goto bail;
> - }
> -
> pend_cnt = dapl_rbuf_count(&evd_ptr->pending_event_queue);
> if (pend_cnt > evd_qlen) {
> spin_unlock(&evd_ptr->header.lock);
> Index: dapl_evd_kcreate.c
> ===================================================================
> --- dapl_evd_kcreate.c (revision 2257)
> +++ dapl_evd_kcreate.c (working copy)
> @@ -96,16 +96,13 @@ DAT_RETURN dapl_evd_kcreate(DAT_IA_HANDL
> goto bail;
> }
>
> - cno_ptr = dapl_cno_alloc (ia_ptr, upcall);
> -
> - if (!cno_ptr)
> - {
> - dat_status = DAT_INSUFFICIENT_RESOURCES;
> - goto bail;
> - }
> -
> - cno_ptr->cno_upcall_policy = upcall_policy;
> + cno_ptr = dapl_cno_alloc (ia_ptr, upcall);
>
> + if (!cno_ptr) {
> + dat_status = DAT_INSUFFICIENT_RESOURCES;
> + goto bail;
> + }
> + cno_ptr->cno_upcall_policy = upcall_policy;
>
> dat_status = dapl_evd_internal_create(ia_ptr,
> cno_ptr,
> @@ -115,9 +112,6 @@ DAT_RETURN dapl_evd_kcreate(DAT_IA_HANDL
> goto bail;
> }
>
> - evd_ptr->evd_state = DAPL_EVD_STATE_OPEN;
> - evd_ptr->evd_enabled = DAT_TRUE;
> -
> evdhandle = *evd_handle = (DAT_EVD_HANDLE) evd_ptr;
>
> bail:
> Index: dapl_openib_util.h
> ===================================================================
> --- dapl_openib_util.h (revision 2257)
> +++ dapl_openib_util.h (working copy)
> @@ -89,22 +89,6 @@ typedef struct ib_hca_transport {
> u16 lid;
> } ib_hca_transport_t;
>
> -/*
> - * ib_shm_transport_t structure. Define fields specific to this
> - * provider implementation necessary to maintain shared memory state
> - *
> - * OpenIB does not have an API to directly register memory as shared
> - * memory. Instead, we must register memory, then register that memory
> - * as shared. This gives us a new mr_handle, as well as a new l_key and
> - * r_key, so we need to save the original handle until it can be
> - * disposed of when the user does an lmr_free. The l_key and r_key are
> - * not available and will disappear when we deallocate the mr_handle, so
> - * we don't save them.
> - */
> -typedef struct ib_shm_transport {
> - ib_mr_handle_t mr_handle;
> -} ib_shm_transport_t;
> -
> #define IB_INVALID_HANDLE NULL
>
> #define IB_MAX_REQ_PDATA_SIZE 92
> Index: dapl_ep_create_with_srq.c
> ===================================================================
> --- dapl_ep_create_with_srq.c (revision 2257)
> +++ dapl_ep_create_with_srq.c (working copy)
> @@ -81,7 +81,6 @@ dapl_ep_create_with_srq(DAT_IA_HANDLE ia
> DAPL_IA *ia_ptr;
> DAPL_EP *ep_ptr;
> DAT_EP_ATTR ep_attr_limit;
> - DAPL_EVD *evd_ptr;
> DAT_RETURN dat_status;
>
> dat_status = DAT_SUCCESS;
> @@ -209,39 +208,6 @@ dapl_ep_create_with_srq(DAT_IA_HANDLE ia
> }
> }
>
> - /*
> - * Verify the completion flags for the EVD and the EP
> - */
> - /*
> - * XXX FIXME
> - * XXX Need to make assign the EVD to the right completion type
> - * XXX depending on the EP attributes. Fail if the types don't
> - * XXX match, they are mutually exclusive.
> - */
> - evd_ptr = (DAPL_EVD *) recv_evd_handle;
> - if (evd_ptr != NULL && evd_ptr->completion_type == DAPL_EVD_STATE_INIT) {
> - if (ep_attr != NULL &&
> - ep_attr->recv_completion_flags ==
> - DAT_COMPLETION_DEFAULT_FLAG) {
> - evd_ptr->completion_type = DAPL_EVD_STATE_THRESHOLD;
> - } else {
> - evd_ptr->completion_type =
> - ep_attr->recv_completion_flags;
> - }
> - }
> -
> - evd_ptr = (DAPL_EVD *) request_evd_handle;
> - if (evd_ptr != NULL && evd_ptr->completion_type == DAPL_EVD_STATE_INIT) {
> - if (ep_attr != NULL &&
> - ep_attr->recv_completion_flags ==
> - DAT_COMPLETION_DEFAULT_FLAG) {
> - evd_ptr->completion_type = DAPL_EVD_STATE_THRESHOLD;
> - } else {
> - evd_ptr->completion_type =
> - ep_attr->recv_completion_flags;
> - }
> - }
> -
> dat_status = DAT_NOT_IMPLEMENTED;
>
> /*
> Index: dapl_evd_util.c
> ===================================================================
> --- dapl_evd_util.c (revision 2257)
> +++ dapl_evd_util.c (working copy)
> @@ -77,9 +77,6 @@ dapl_evd_internal_create(DAPL_IA * ia_pt
> evd_ptr->evd_producer_locking_needed =
> ((evd_flags & ~(DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG)) != 0);
>
> - /* Before we setup any callbacks, transition state to OPEN. */
> - evd_ptr->evd_state = DAPL_EVD_STATE_OPEN;
> -
> if (evd_flags & DAT_EVD_ASYNC_FLAG) {
> /*
> * There is no cq associate with async evd. Set it to invalid
> @@ -161,19 +158,13 @@ DAPL_EVD *dapl_evd_alloc(DAPL_IA * ia_pt
> /*
> * Initialize the body
> */
> - evd_ptr->evd_state = DAPL_EVD_STATE_INITIAL;
> + evd_ptr->evd_state = DAPL_EVD_STATE_OPEN;
> evd_ptr->evd_flags = evd_flags;
> - evd_ptr->evd_enabled = DAT_TRUE;
> - evd_ptr->evd_waitable = DAT_TRUE;
> evd_ptr->evd_producer_locking_needed = 1; /* Conservative value. */
> evd_ptr->ib_cq_handle = IB_INVALID_HANDLE;
> atomic_set(&evd_ptr->evd_ref_count, 0);
> evd_ptr->catastrophic_overflow = DAT_FALSE;
> evd_ptr->qlen = qlen;
> - evd_ptr->completion_type = DAPL_EVD_STATE_THRESHOLD; /* FIXME: should be DAPL_EVD_STATE_INIT */
> - dapl_os_wait_object_init(&evd_ptr->wait_object);
> -
> - evd_ptr->cno_active_count = 0;
> evd_ptr->cno_ptr = cno_ptr;
>
> bail:
> @@ -219,10 +210,6 @@ DAT_RETURN dapl_evd_event_alloc(DAPL_EVD
> event_ptr++;
> }
>
> - evd_ptr->cq_notified = DAT_FALSE;
> - evd_ptr->cq_notified_when = 0;
> - evd_ptr->threshold = 0;
> -
> bail:
> return dat_status;
> }
> @@ -331,7 +318,6 @@ DAT_RETURN dapl_evd_dealloc(DAPL_EVD * e
> kfree(evd_ptr->events);
> }
>
> - dapl_os_wait_object_destroy(&evd_ptr->wait_object);
> kfree(evd_ptr);
>
> bail:
> @@ -445,73 +431,20 @@ static void dapl_evd_post_event(DAPL_EVD
> (void *)event_ptr);
> dapl_os_assert(dat_status == DAT_SUCCESS);
>
> - dapl_os_assert(evd_ptr->evd_state == DAPL_EVD_STATE_WAITED
> - || evd_ptr->evd_state == DAPL_EVD_STATE_OPEN);
> + dapl_os_assert(evd_ptr->evd_state == DAPL_EVD_STATE_OPEN);
>
> if (evd_ptr->evd_state == DAPL_EVD_STATE_OPEN) {
> /* No waiter. Arrange to trigger a CNO if it exists. */
> -
> - if (evd_ptr->evd_enabled) {
> - cno_to_trigger = evd_ptr->cno_ptr;
> - }
> -
> - if (evd_ptr->evd_producer_locking_needed) {
> - spin_unlock(&evd_ptr->header.lock);
> - }
> - } else {
> - /*
> - * We're in DAPL_EVD_STATE_WAITED. Take the lock if
> - * we don't have it, recheck, and signal.
> - */
> - if (!evd_ptr->evd_producer_locking_needed) {
> - spin_lock(&evd_ptr->header.lock);
> - }
> -
> - if (evd_ptr->evd_state == DAPL_EVD_STATE_WAITED
> - && (dapl_rbuf_count(&evd_ptr->pending_event_queue)
> - >= evd_ptr->threshold)) {
> - spin_unlock(&evd_ptr->header.lock);
> - dapl_os_wait_object_wakeup(&evd_ptr->wait_object);
> - } else {
> - spin_unlock(&evd_ptr->header.lock);
> - }
> + cno_to_trigger = evd_ptr->cno_ptr;
> + if (evd_ptr->evd_producer_locking_needed) {
> + spin_unlock(&evd_ptr->header.lock);
> + }
> }
> -
> if (cno_to_trigger != NULL) {
> dapl_cno_trigger (cno_to_trigger, evd_ptr);
> }
> }
>
> -/*
> - * Post the <event> to the evd. Do not do any wakeup processing.
> - * This function should only be called if it is known that there are
> - * no waiters that it is appropriate to wakeup on this EVD. An example
> - * of such a situation is during internal dat_evd_wait() processing.
> - *
> - * If producer side locking is required, the EVD lock must be held upon
> - * entry to this function.
> - */
> -
> -static void
> -dapl_evd_post_event_nosignal(DAPL_EVD * evd_ptr, const DAT_EVENT * event_ptr)
> -{
> - DAT_RETURN dat_status;
> -
> - dapl_dbg_log(DAPL_DBG_TYPE_EVD,
> - "dapl_evd_post_event_nonsignal: Called with event # %x\n",
> - event_ptr->event_number);
> -
> - dat_status = dapl_rbuf_add(&evd_ptr->pending_event_queue,
> - (void *)event_ptr);
> - dapl_os_assert(dat_status == DAT_SUCCESS);
> -
> - dapl_os_assert(evd_ptr->evd_state == DAPL_EVD_STATE_WAITED
> - || evd_ptr->evd_state == DAPL_EVD_STATE_OPEN);
> -
> - if (evd_ptr->evd_producer_locking_needed) {
> - spin_unlock(&evd_ptr->header.lock);
> - }
> -}
>
> /*
> * format an overflow event for posting
> @@ -689,49 +622,6 @@ dapl_evd_post_software_event(DAPL_EVD *
> }
>
> /*
> - * dapl_evd_post_generic_event
> - *
> - * Post a generic event type. Not used by all providers
> - *
> - * Input:
> - * evd_ptr
> - * event_number
> - * data
> - *
> - * Output:
> - * none
> - *
> - * Returns:
> - * DAT_SUCCESS
> - *
> - */
> -DAT_RETURN
> -dapl_evd_post_generic_event(DAPL_EVD * evd_ptr,
> - DAT_EVENT_NUMBER event_number,
> - DAT_EVENT_DATA * data)
> -{
> - DAT_EVENT *event_ptr;
> -
> - event_ptr = dapl_evd_get_and_init_event(evd_ptr, event_number);
> - /*
> - * Note event lock may be held on successful return
> - * to be released by dapl_evd_post_event(), if provider side locking
> - * is needed.
> - */
> -
> - if (event_ptr == NULL) {
> - return DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
> - DAT_RESOURCE_MEMORY);
> - }
> -
> - event_ptr->event_data = *data;
> -
> - dapl_evd_post_event(evd_ptr, event_ptr);
> -
> - return DAT_SUCCESS;
> -}
> -
> -/*
> * dapl_evd_cqe_to_event
> *
> * Convert a CQE into an event structure.
> @@ -866,128 +756,13 @@ dapl_evd_cqe_to_event(DAPL_EVD * evd_ptr
> }
> } /* end switch */
>
> - /*
> - * Most error DTO ops result in disconnecting the EP. See
> - * IBTA Vol 1.1, Chapter 10,Table 68, for expected effect on
> - * state.
> - */
> - if ((dto_status != DAT_DTO_SUCCESS) &&
> - (dto_status != DAT_DTO_ERR_FLUSHED)) {
> -#ifdef CONN_EV_BROKEN_SUPPORTED
> - DAPL_EVD *evd_ptr;
> -
> - /*
> - * If we are connected, generate disconnect and generate an
> - * event. We may be racing with other disconnect ops, so we
> - * need to check. We may also be racing CM connection events,
> - * requiring us to check for connection pending states too.
> - */
> - spin_lock(&ep_ptr->header.lock);
> - if (ep_ptr->param.ep_state == DAT_EP_STATE_CONNECTED ||
> - ep_ptr->param.ep_state ==
> - DAT_EP_STATE_ACTIVE_CONNECTION_PENDING
> - || ep_ptr->param.ep_state ==
> - DAT_EP_STATE_PASSIVE_CONNECTION_PENDING
> - || ep_ptr->param.ep_state ==
> - DAT_EP_STATE_COMPLETION_PENDING) {
> - ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED;
> - spin_unlock(&ep_ptr->header.lock);
> -
> - /* Let the other side know we have disconnected */
> - // cant call to dapl_ib_disconnect besause it is in interrupt context
> - // and this call is blocking need to fix ???
> - // (void) dapl_ib_disconnect (ep_ptr, DAT_CLOSE_ABRUPT_FLAG);
> - printk("FIXME DAPL BUG %s %s %d \n", __FILE__,
> - __FUNCTION__, __LINE__);
> - /* ... and clean up the local side */
> - evd_ptr = (DAPL_EVD *) ep_ptr->param.connect_evd_handle;
> - if (evd_ptr != NULL) {
> - dapl_evd_post_connection_event(evd_ptr,
> - DAT_CONNECTION_EVENT_BROKEN,
> - (DAT_HANDLE)
> - ep_ptr, 0, 0);
> - }
> - } else {
> - spin_unlock(&ep_ptr->header.lock);
> - }
> -
> - dapl_dbg_log(DAPL_DBG_TYPE_DTO_COMP_ERR,
> - " DTO completion ERROR: %d: op %#x (ep disconnected)\n",
> - DAPL_GET_CQE_STATUS(cqe_ptr),
> - DAPL_GET_CQE_OPTYPE(cqe_ptr));
> -#else
> - dapl_dbg_log(DAPL_DBG_TYPE_DTO_COMP_ERR,
> - " DTO completion ERROR: %d: op %#x\n",
> - DAPL_GET_CQE_STATUS(cqe_ptr),
> - DAPL_GET_CQE_OPTYPE(cqe_ptr));
> -#endif
> - }
> -}
> -
> -/*
> - * dapl_evd_copy_cq
> - *
> - * Copy all entries on a CQ associated with the EVD onto that EVD
> - * Up to caller to handle races, if any. Note that no EVD waiters will
> - * be awoken by this copy.
> - *
> - * Input:
> - * evd_ptr
> - *
> - * Output:
> - * None
> - *
> - * Returns:
> - * none
> - *
> - */
> -void dapl_evd_copy_cq(DAPL_EVD * evd_ptr)
> -{
> - ib_work_completion_t cur_cqe;
> - DAT_RETURN dat_status;
> - DAT_EVENT *event;
> -
> - if (evd_ptr->ib_cq_handle == IB_INVALID_HANDLE) {
> - /* Nothing to do if no CQ. */
> - return;
> - }
> -
> - while (1) {
> - dat_status =
> - dapl_ib_completion_poll(evd_ptr->header.owner_ia->hca_ptr,
> - evd_ptr, &cur_cqe);
> -
> - if (dat_status != DAT_SUCCESS) {
> - break;
> - }
> -
> - /* For debugging. */
> - dapl_evd_eh_print_cqe(&cur_cqe);
> -
> - /*
> - * Can use DAT_DTO_COMPLETION_EVENT because dapl_evd_cqe_to_event
> - * will overwrite.
> - */
> -
> - event =
> - dapl_evd_get_and_init_event(evd_ptr,
> - DAT_DTO_COMPLETION_EVENT);
> - if (event == NULL) {
> - /* We've already attempted the overflow post; return. */
> - return;
> - }
> -
> - dapl_evd_cqe_to_event(evd_ptr, &cur_cqe, event);
> -
> - dapl_evd_post_event_nosignal(evd_ptr, event);
> - }
> -
> - if (DAT_GET_TYPE(dat_status) != DAT_QUEUE_EMPTY) {
> - dapl_dbg_log(DAPL_DBG_TYPE_EVD,
> - "dapl_evd_copy_cq: dapl_ib_completion_poll returned 0x%x\n",
> - dat_status);
> - dapl_os_assert(!"Bad return from dapl_ib_completion_poll");
> - }
> + if (dto_status != DAT_DTO_SUCCESS) {
> + dapl_dbg_log(DAPL_DBG_TYPE_DTO_COMP_ERR,
> + " DTO completion ERROR:status %d: op %#x vendore synd: 0x%x\n",
> + DAPL_GET_CQE_STATUS (cqe_ptr),
> + DAPL_GET_CQE_OPTYPE (cqe_ptr),
> + DAPL_GET_CQE_VND_SYND(cqe_ptr));
> + }
> }
>
> /*
> Index: dapl_evd_util.h
> ===================================================================
> --- dapl_evd_util.h (revision 2257)
> +++ dapl_evd_util.h (working copy)
> @@ -87,10 +87,6 @@ DAT_RETURN
> dapl_evd_post_software_event(DAPL_EVD * evd_ptr,
> DAT_EVENT_NUMBER event_number, DAT_PVOID pointer);
>
> -DAT_RETURN
> -dapl_evd_post_generic_event(DAPL_EVD * evd_ptr,
> - DAT_EVENT_NUMBER event_number,
> - DAT_EVENT_DATA * data);
>
> /*************************************
> * dapl internal callbacks functions *
> @@ -115,8 +111,6 @@ extern void dapl_evd_cq_async_error_call
> extern void dapl_evd_qp_async_error_callback(ib_error_record_t * cause_ptr,
> void *context);
>
> -extern void dapl_evd_copy_cq(DAPL_EVD * evd_ptr);
> -
> extern DAT_RETURN dapl_evd_cq_poll_to_event(DAPL_EVD * evd_ptr,
> DAT_EVENT * event);
>
> Index: dapl.h
> ===================================================================
> --- dapl.h (revision 2257)
> +++ dapl.h (working copy)
> @@ -64,10 +64,7 @@ typedef enum dapl_magic {
> } DAPL_MAGIC;
>
> typedef enum dapl_evd_state {
> - DAPL_EVD_STATE_TERMINAL,
> - DAPL_EVD_STATE_INITIAL,
> DAPL_EVD_STATE_OPEN,
> - DAPL_EVD_STATE_WAITED,
> DAPL_EVD_STATE_DEAD = 0xDEAD
> } DAPL_EVD_STATE;
>
> @@ -270,8 +267,6 @@ struct dapl_evd
>
> DAPL_EVD_STATE evd_state;
> DAT_EVD_FLAGS evd_flags;
> - DAT_BOOLEAN evd_enabled; /* For attached CNO. */
> - DAT_BOOLEAN evd_waitable; /* EVD state. */
>
> /* Derived from evd_flags; see dapls_evd_internal_create. */
> DAT_BOOLEAN evd_producer_locking_needed;
> @@ -279,9 +274,6 @@ struct dapl_evd
> /* Every EVD has a CQ unless it is a SOFTWARE_EVENT only EVD */
> ib_cq_handle_t ib_cq_handle;
>
> - /* Mellanox Specific completion handle for registration/de-registration */
> - ib_comp_handle_t ib_comp_handle;
> -
> /* An Event Dispatcher cannot be freed while
> * it is referenced elsewhere.
> */
> @@ -295,20 +287,7 @@ struct dapl_evd
> DAT_EVENT *events;
> DAPL_RING_BUFFER free_event_queue;
> DAPL_RING_BUFFER pending_event_queue;
> -
> - /* CQ Completions are not placed into 'deferred_events'
> - ** rather they are simply left on the Completion Queue
> - ** and the fact that there was a notification is flagged.
> - */
> - DAT_BOOLEAN cq_notified;
> - DAPL_OS_TICKS cq_notified_when;
> -
> - DAT_COUNT cno_active_count;
> DAPL_CNO *cno_ptr;
> -
> - DAPL_OS_WAIT_OBJECT wait_object;
> - DAT_COUNT threshold;
> - DAPL_EVD_COMPLETION completion_type;
> };
>
> /* uDAPL timer entry, used to queue timeouts */
> --
> Itamar
>
More information about the general
mailing list