[Openib-windows] RE: [PATCH] query-cq
Yossi Leybovich
sleybo at mellanox.co.il
Sun Sep 11 01:30:17 PDT 2005
Fab
I am Ok with that,I test it and it work.
One small thing,
I think you forgot to patch 2 files (interface files)
Index:
C:/sleybo/latest/hw/mt23108/vapi/Hca/hcahal/tavor/thhul_hob/thhul_hob.c
===================================================================
--- C:/sleybo/latest/hw/mt23108/vapi/Hca/hcahal/tavor/thhul_hob/thhul_hob.c
(revision 351)
+++ C:/sleybo/latest/hw/mt23108/vapi/Hca/hcahal/tavor/thhul_hob/thhul_hob.c
(working copy)
@@ -86,6 +86,7 @@
NULL /* HHULIF_poll4cqe */,
NULL /* HHULIF_poll_and_rearm_cq */,
THHUL_cqm_peek_cq /* HHULIF_peek_cq */,
+ THHUL_cqm_query_cq /* HHULIF_query_cq */,
THHUL_cqm_req_comp_notif /* HHULIF_req_comp_notif */,
THHUL_cqm_req_ncomp_notif /* HHULIF_req_ncomp_notif */,
THHUL_cqm_destroy_cq_done /* HHULIF_destroy_cq_done */,
Index: C:/sleybo/latest/hw/mt23108/vapi/Hca/hcahal/hhul.h
===================================================================
--- C:/sleybo/latest/hw/mt23108/vapi/Hca/hcahal/hhul.h (revision 351)
+++ C:/sleybo/latest/hw/mt23108/vapi/Hca/hcahal/hhul.h (working copy)
@@ -229,6 +229,10 @@
VAPI_cqe_num_t cqe_num);
+ HH_ret_t (*HHULIF_query_cq)(HHUL_hca_hndl_t hca_hndl,
+ HHUL_cq_hndl_t cq,
+ VAPI_cqe_num_t* num_o_cqes_p);
+
HH_ret_t (*HHULIF_req_comp_notif)(HHUL_hca_hndl_t hca_hndl,
HHUL_cq_hndl_t cq,
VAPI_cq_notif_type_t notif_type);
> -----Original Message-----
> From: Fab Tillier [mailto:ftillier at silverstorm.com]
> Sent: Thursday, September 08, 2005 8:18 PM
> To: 'Yossi Leybovich'
> Cc: openib-windows at openib.org; Ziv Farjun
> Subject: RE: [PATCH] query-cq
>
>
> > From: Yossi Leybovich [mailto:sleybo at mellanox.co.il]
> > Sent: Wednesday, September 07, 2005 4:59 AM
> >
> > The issue is that for UL CQ the kernel get the final size so the
> > query shouldn't move to kernel but finish in the user level (the
> > kernel does not know about the spare cqes)
>
> Ahh, I didn't realize only THHUL knew about spare CQEs. Ok, this all
> makes sense now.
>
> > I attached patch that return the correct size also in CQs that were
> > created in kernel.(+ the previous changes)
>
> I made some slight changes to the patch. Most significantly, I
> changed the pre-query call to return the size if it returns
> IB_VERBS_PROCESSING_DONE. This allowed eliminating the post-query
> call. This is something that I'll cleanup on a larger scale at some
> point - there is no reason to have the post_xxx calls if the UVP
> finished processing in the pre_xxx call.
>
> > I didn't manage to test it because I could not make alts work in
> > kernel ( BTW Is alts work at kernel level ? How ? )
>
> ALTS can be run in kernel mode, but it's a pain. I had hacked it at
> some point to load as an upper filter driver to the HCA driver, and
> run the tests after the START_DEVICE PnP IRP finished being processed
> by the lower driver. There are better ways of doing this by making
> the kernel ALTS a service and using the service control manager to
> start and stop the driver, as well as to launch tests. This hasn't
> been a priority, though. I changed one minor thing in the kernel
> driver - the check for the umv_buf command is moved to the
> head of the query_cq function, and if there's a command the
> call is failed. Since the UVP is expected to handle this
> entirely in user-mode, I thought it might make sense to fail
> this explicitly. In any case, the kernel code looks good
> enough that I'd be comfortable checking in.
>
> Let me know what you think, and if it looks good enough to check in.
>
> Thanks,
>
> - Fab
>
> Index: core/al/user/ual_cq.c
> ===================================================================
> --- core/al/user/ual_cq.c (revision 61)
> +++ core/al/user/ual_cq.c (working copy)
> @@ -305,7 +305,13 @@
> if( h_cq->h_ci_cq && uvp_intf.pre_query_cq )
> {
> /* Pre call to the UVP library */
> - status = uvp_intf.pre_query_cq( h_cq->h_ci_cq,
> &cq_ioctl.in.umv_buf );
> + status = uvp_intf.pre_query_cq(
> + h_cq->h_ci_cq, p_size, &cq_ioctl.in.umv_buf );
> + if( status == IB_VERBS_PROCESSING_DONE )
> + {
> + AL_EXIT( AL_DBG_CQ );
> + return IB_SUCCESS;
> + }
> if( status != IB_SUCCESS )
> {
> AL_EXIT( AL_DBG_CQ );
> Index: inc/user/iba/ib_uvp.h
> ===================================================================
> --- inc/user/iba/ib_uvp.h (revision 61)
> +++ inc/user/iba/ib_uvp.h (working copy)
> @@ -1739,6 +1739,7 @@
> typedef ib_api_status_t
> (AL_API *uvp_pre_query_cq) (
> IN const ib_cq_handle_t
> h_uvp_cq,
> + IN OUT uint32_t* const
> p_size,
> IN OUT ci_umv_buf_t
> *p_umv_buf );
>
> /*
> @@ -1752,6 +1753,9 @@
> * h_uvp_cq
> * [in] Vendor's Handle to the already created CQ
> (in user-mode
> library).
> *
> +* p_size
> +* [out] Size of the CQ if processing ends in user-mode.
> +*
> * p_umv_buf
> * [in out] On input, UAL provides this buffer template.
> * On return from this function, p_umv_buf contains
> @@ -1765,6 +1769,9 @@
> * The CQ handle is invalid.
> * IB_INSUFFICIENT_RESOURCES
> * Insufficient resources in Vendor library to
> complete the call.
> +* IB_VERBS_PROCESSING_DONE
> +* The UVP fully processed the request. The
> post_query_cq handler
> +* will not be invoked.
> *
> * PORTABILITY
> * User mode.
> Index: hw/mt23108/vapi/Hca/hcahal/tavor/thhul_cqm/thhul_cqm.c
> ===================================================================
> --- hw/mt23108/vapi/Hca/hcahal/tavor/thhul_cqm/thhul_cqm.c
> (revision 61)
> +++ hw/mt23108/vapi/Hca/hcahal/tavor/thhul_cqm/thhul_cqm.c
> (working copy)
> @@ -1547,7 +1547,7 @@
>
> /* parameters checks */
> if (cq_p == NULL) {
> - MTL_ERROR1("THHUL_cqm_peek_cq: NULL CQ handle.\n");
> + MTL_ERROR1("THHUL_cqm_count_cqe: NULL CQ handle.\n");
> return IB_INVALID_CQ_HANDLE;
> }
>
> @@ -1608,6 +1608,29 @@
> }
>
>
> +HH_ret_t THHUL_cqm_query_cq(
> + /*IN*/ HHUL_hca_hndl_t hca_hndl,
> + /*IN*/ HHUL_cq_hndl_t cq,
> + /*OUT*/ VAPI_cqe_num_t *num_o_cqes_p)
> +{
> + THHUL_cq_t *cq_p= (THHUL_cq_t*)cq;
> + HH_ret_t ret=HH_OK;
> +
> + /* parameters checks */
> + if (cq_p == NULL) {
> + MTL_ERROR1("THHUL_cqm_query_cq: NULL CQ handle.\n");
> + return HH_EINVAL_CQ_HNDL;
> + }
> +
> + /* Find CQE and check ownership */
> + MOSAL_spinlock_dpc_lock(&(cq_p->cq_lock));
> + *num_o_cqes_p= ((1U << cq_p->cur_buf.log2_num_o_cqes) -1 -
> cq_p->cur_buf.spare_cqes) ;
> +
> + MOSAL_spinlock_unlock(&(cq_p->cq_lock));
> +
> + return ret;
> +}
> +
> static void rearm_cq(THHUL_cq_t *cq_p, MT_bool
> solicitedNotification) {
> volatile u_int32_t chimeWords[2];
> THH_uar_t uar = cq_p->uar;
> Index: hw/mt23108/vapi/Hca/hcahal/tavor/thhul_cqm/thhul_cqm.h
> ===================================================================
> --- hw/mt23108/vapi/Hca/hcahal/tavor/thhul_cqm/thhul_cqm.h
> (revision 61)
> +++ hw/mt23108/vapi/Hca/hcahal/tavor/thhul_cqm/thhul_cqm.h
> (working copy)
> @@ -110,6 +110,12 @@
> OUT uint32_t* const
> p_n_cqes );
> #endif
>
> +DLL_API HH_ret_t THHUL_cqm_query_cq(
> + /*IN*/ HHUL_hca_hndl_t hca_hndl,
> + /*IN*/ HHUL_cq_hndl_t cq,
> + /*OUT*/ VAPI_cqe_num_t *num_o_cqes_p
> +);
> +
> DLL_API HH_ret_t THHUL_cqm_peek_cq(
> /*IN*/ HHUL_hca_hndl_t hca_hndl,
> /*IN*/ HHUL_cq_hndl_t cq,
> Index: hw/mt23108/kernel/hca_verbs.c
> ===================================================================
> --- hw/mt23108/kernel/hca_verbs.c (revision 61)
> +++ hw/mt23108/kernel/hca_verbs.c (working copy)
> @@ -2130,6 +2130,14 @@
> status = IB_INVALID_PARAMETER;
> goto cleanup;
> }
> +
> + /* Query is fully handled in user-mode. */
> + if( p_umv_buf && p_umv_buf->command )
> + {
> + status = IB_INVALID_CQ_HANDLE;
> + goto cleanup;
> + }
> +
> VALIDATE_INDEX(hca_idx, MLNX_MAX_HCA,
> IB_INVALID_CQ_HANDLE, cleanup);
> hobul_p = mlnx_hobul_array[hca_idx];
> if (NULL == hobul_p) {
> @@ -2147,19 +2155,13 @@
> cl_mutex_acquire(&hobul_p->cq_info_tbl[cq_idx].mutex);
>
> hhul_cq_hndl = hobul_p->cq_info_tbl[cq_idx].hhul_cq_hndl;
> -
> - if (HH_OK != THH_hob_query_cq(hobul_p->hh_hndl, cq_num,
> p_size)) {
> + if (HH_OK != THHUL_cqm_query_cq(hobul_p->hhul_hndl,
> hhul_cq_hndl,
> p_size)){
> status = IB_ERROR;
> goto cleanup_locked;
> }
>
> cl_mutex_release(&hobul_p->cq_info_tbl[cq_idx].mutex);
>
> - if( p_umv_buf && p_umv_buf->command )
> - {
> - p_umv_buf->output_size = 0;
> - p_umv_buf->status = IB_SUCCESS;
> - }
> CL_EXIT(MLNX_DBG_TRACE, g_mlnx_dbg_lvl);
> return IB_SUCCESS;
>
> Index: hw/mt23108/user/mlnx_ual_cq.c
> ===================================================================
> --- hw/mt23108/user/mlnx_ual_cq.c (revision 61)
> +++ hw/mt23108/user/mlnx_ual_cq.c (working copy)
> @@ -48,7 +48,7 @@
> p_uvp->pre_create_cq = mlnx_pre_create_cq;
> p_uvp->post_create_cq = mlnx_post_create_cq;
>
> - p_uvp->pre_query_cq = NULL;
> + p_uvp->pre_query_cq = mlnx_pre_query_cq;
> p_uvp->post_query_cq = NULL;
>
> p_uvp->pre_resize_cq = mlnx_pre_resize_cq;
> @@ -434,26 +434,18 @@
>
> ib_api_status_t
> mlnx_pre_query_cq (
> - IN const ib_cq_handle_t h_uvp_cq,
> - IN OUT ci_umv_buf_t *p_umv_buf)
> + IN const ib_cq_handle_t h_uvp_cq,
> + OUT uint32_t* const p_size,
> + IN OUT ci_umv_buf_t
> *p_umv_buf)
> {
> - FUNC_ENTER;
> - CL_ASSERT(p_umv_buf);
> - p_umv_buf->command = TRUE;
> - FUNC_EXIT;
> - return IB_SUCCESS;
> -}
> + mlnx_ual_cq_info_t *p_cq_info = (mlnx_ual_cq_info_t *)((void*)
> h_uvp_cq);
>
> + FUNC_ENTER;
>
> -void
> -mlnx_post_query_cq (
> - IN const ib_cq_handle_t h_uvp_cq,
> - IN ib_api_status_t ioctl_status,
> - IN OUT ci_umv_buf_t *p_umv_buf)
> -{
> - FUNC_ENTER;
> - FUNC_EXIT;
> - return;
> + *p_size = p_cq_info->cq_size;
> +
> + FUNC_EXIT;
> + return IB_VERBS_PROCESSING_DONE;
> }
>
>
> Index: hw/mt23108/user/mlnx_ual_main.h
> ===================================================================
> --- hw/mt23108/user/mlnx_ual_main.h (revision 61)
> +++ hw/mt23108/user/mlnx_ual_main.h (working copy)
> @@ -250,15 +250,10 @@
>
> ib_api_status_t
> mlnx_pre_query_cq (
> - IN const ib_cq_handle_t h_uvp_cq,
> - IN OUT ci_umv_buf_t *p_umv_buf);
> + IN const ib_cq_handle_t h_uvp_cq,
> + OUT uint32_t* const p_size,
> + IN OUT ci_umv_buf_t
> *p_umv_buf);
>
> -void
> -mlnx_post_query_cq (
> - IN const ib_cq_handle_t h_uvp_cq,
> - IN ib_api_status_t
> ioctl_status,
> - IN OUT ci_umv_buf_t *p_umv_buf);
> -
> ib_api_status_t
> mlnx_pre_destroy_cq (
> IN const ib_cq_handle_t h_uvp_cq);
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20050911/7a50fbc4/attachment.html>
-------------- next part --------------
An embedded and charset-unspecified text was scrubbed...
Name: add1_query_cq.diff.txt
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20050911/7a50fbc4/attachment.txt>
-------------- next part --------------
An embedded and charset-unspecified text was scrubbed...
Name: add_query_cq.diff.txt
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20050911/7a50fbc4/attachment-0001.txt>
More information about the ofw
mailing list