[Openib-windows] RE: [PATCH] query-cq
Fab Tillier
ftillier at silverstorm.com
Thu Sep 8 10:18:10 PDT 2005
> From: Yossi Leybovich [mailto:sleybo at mellanox.co.il]
> Sent: Wednesday, September 07, 2005 4:59 AM
>
> The issue is that for UL CQ the kernel get the final size
> so the query shouldn't move to kernel but finish in the user level (the kernel
> does not know about the spare cqes)
Ahh, I didn't realize only THHUL knew about spare CQEs. Ok, this all makes
sense now.
> I attached patch that return the correct size also in CQs that were created in
> kernel.(+ the previous changes)
I made some slight changes to the patch. Most significantly, I changed the
pre-query call to return the size if it returns IB_VERBS_PROCESSING_DONE. This
allowed eliminating the post-query call. This is something that I'll cleanup on
a larger scale at some point - there is no reason to have the post_xxx calls if
the UVP finished processing in the pre_xxx call.
> I didn't manage to test it because I could not make alts work in kernel ( BTW
> Is alts work at kernel level ? How ? )
ALTS can be run in kernel mode, but it's a pain. I had hacked it at some point
to load as an upper filter driver to the HCA driver, and run the tests after the
START_DEVICE PnP IRP finished being processed by the lower driver. There are
better ways of doing this by making the kernel ALTS a service and using the
service control manager to start and stop the driver, as well as to launch
tests. This hasn't been a priority, though. I changed one minor thing in the
kernel driver - the check for the umv_buf command is moved to the head of the
query_cq function, and if there's a command the call is failed. Since the UVP
is expected to handle this entirely in user-mode, I thought it might make sense
to fail this explicitly. In any case, the kernel code looks good enough that
I'd be comfortable checking in.
Let me know what you think, and if it looks good enough to check in.
Thanks,
- Fab
Index: core/al/user/ual_cq.c
===================================================================
--- core/al/user/ual_cq.c (revision 61)
+++ core/al/user/ual_cq.c (working copy)
@@ -305,7 +305,13 @@
if( h_cq->h_ci_cq && uvp_intf.pre_query_cq )
{
/* Pre call to the UVP library */
- status = uvp_intf.pre_query_cq( h_cq->h_ci_cq,
&cq_ioctl.in.umv_buf );
+ status = uvp_intf.pre_query_cq(
+ h_cq->h_ci_cq, p_size, &cq_ioctl.in.umv_buf );
+ if( status == IB_VERBS_PROCESSING_DONE )
+ {
+ AL_EXIT( AL_DBG_CQ );
+ return IB_SUCCESS;
+ }
if( status != IB_SUCCESS )
{
AL_EXIT( AL_DBG_CQ );
Index: inc/user/iba/ib_uvp.h
===================================================================
--- inc/user/iba/ib_uvp.h (revision 61)
+++ inc/user/iba/ib_uvp.h (working copy)
@@ -1739,6 +1739,7 @@
typedef ib_api_status_t
(AL_API *uvp_pre_query_cq) (
IN const ib_cq_handle_t
h_uvp_cq,
+ IN OUT uint32_t* const
p_size,
IN OUT ci_umv_buf_t
*p_umv_buf );
/*
@@ -1752,6 +1753,9 @@
* h_uvp_cq
* [in] Vendor's Handle to the already created CQ (in user-mode
library).
*
+* p_size
+* [out] Size of the CQ if processing ends in user-mode.
+*
* p_umv_buf
* [in out] On input, UAL provides this buffer template.
* On return from this function, p_umv_buf contains
@@ -1765,6 +1769,9 @@
* The CQ handle is invalid.
* IB_INSUFFICIENT_RESOURCES
* Insufficient resources in Vendor library to complete the call.
+* IB_VERBS_PROCESSING_DONE
+* The UVP fully processed the request. The post_query_cq handler
+* will not be invoked.
*
* PORTABILITY
* User mode.
Index: hw/mt23108/vapi/Hca/hcahal/tavor/thhul_cqm/thhul_cqm.c
===================================================================
--- hw/mt23108/vapi/Hca/hcahal/tavor/thhul_cqm/thhul_cqm.c (revision 61)
+++ hw/mt23108/vapi/Hca/hcahal/tavor/thhul_cqm/thhul_cqm.c (working copy)
@@ -1547,7 +1547,7 @@
/* parameters checks */
if (cq_p == NULL) {
- MTL_ERROR1("THHUL_cqm_peek_cq: NULL CQ handle.\n");
+ MTL_ERROR1("THHUL_cqm_count_cqe: NULL CQ handle.\n");
return IB_INVALID_CQ_HANDLE;
}
@@ -1608,6 +1608,29 @@
}
+HH_ret_t THHUL_cqm_query_cq(
+ /*IN*/ HHUL_hca_hndl_t hca_hndl,
+ /*IN*/ HHUL_cq_hndl_t cq,
+ /*OUT*/ VAPI_cqe_num_t *num_o_cqes_p)
+{
+ THHUL_cq_t *cq_p= (THHUL_cq_t*)cq;
+ HH_ret_t ret=HH_OK;
+
+ /* parameters checks */
+ if (cq_p == NULL) {
+ MTL_ERROR1("THHUL_cqm_query_cq: NULL CQ handle.\n");
+ return HH_EINVAL_CQ_HNDL;
+ }
+
+ /* Find CQE and check ownership */
+ MOSAL_spinlock_dpc_lock(&(cq_p->cq_lock));
+ *num_o_cqes_p= ((1U << cq_p->cur_buf.log2_num_o_cqes) -1 -
cq_p->cur_buf.spare_cqes) ;
+
+ MOSAL_spinlock_unlock(&(cq_p->cq_lock));
+
+ return ret;
+}
+
static void rearm_cq(THHUL_cq_t *cq_p, MT_bool solicitedNotification) {
volatile u_int32_t chimeWords[2];
THH_uar_t uar = cq_p->uar;
Index: hw/mt23108/vapi/Hca/hcahal/tavor/thhul_cqm/thhul_cqm.h
===================================================================
--- hw/mt23108/vapi/Hca/hcahal/tavor/thhul_cqm/thhul_cqm.h (revision 61)
+++ hw/mt23108/vapi/Hca/hcahal/tavor/thhul_cqm/thhul_cqm.h (working copy)
@@ -110,6 +110,12 @@
OUT uint32_t* const
p_n_cqes );
#endif
+DLL_API HH_ret_t THHUL_cqm_query_cq(
+ /*IN*/ HHUL_hca_hndl_t hca_hndl,
+ /*IN*/ HHUL_cq_hndl_t cq,
+ /*OUT*/ VAPI_cqe_num_t *num_o_cqes_p
+);
+
DLL_API HH_ret_t THHUL_cqm_peek_cq(
/*IN*/ HHUL_hca_hndl_t hca_hndl,
/*IN*/ HHUL_cq_hndl_t cq,
Index: hw/mt23108/kernel/hca_verbs.c
===================================================================
--- hw/mt23108/kernel/hca_verbs.c (revision 61)
+++ hw/mt23108/kernel/hca_verbs.c (working copy)
@@ -2130,6 +2130,14 @@
status = IB_INVALID_PARAMETER;
goto cleanup;
}
+
+ /* Query is fully handled in user-mode. */
+ if( p_umv_buf && p_umv_buf->command )
+ {
+ status = IB_INVALID_CQ_HANDLE;
+ goto cleanup;
+ }
+
VALIDATE_INDEX(hca_idx, MLNX_MAX_HCA, IB_INVALID_CQ_HANDLE, cleanup);
hobul_p = mlnx_hobul_array[hca_idx];
if (NULL == hobul_p) {
@@ -2147,19 +2155,13 @@
cl_mutex_acquire(&hobul_p->cq_info_tbl[cq_idx].mutex);
hhul_cq_hndl = hobul_p->cq_info_tbl[cq_idx].hhul_cq_hndl;
-
- if (HH_OK != THH_hob_query_cq(hobul_p->hh_hndl, cq_num, p_size)) {
+ if (HH_OK != THHUL_cqm_query_cq(hobul_p->hhul_hndl, hhul_cq_hndl,
p_size)){
status = IB_ERROR;
goto cleanup_locked;
}
cl_mutex_release(&hobul_p->cq_info_tbl[cq_idx].mutex);
- if( p_umv_buf && p_umv_buf->command )
- {
- p_umv_buf->output_size = 0;
- p_umv_buf->status = IB_SUCCESS;
- }
CL_EXIT(MLNX_DBG_TRACE, g_mlnx_dbg_lvl);
return IB_SUCCESS;
Index: hw/mt23108/user/mlnx_ual_cq.c
===================================================================
--- hw/mt23108/user/mlnx_ual_cq.c (revision 61)
+++ hw/mt23108/user/mlnx_ual_cq.c (working copy)
@@ -48,7 +48,7 @@
p_uvp->pre_create_cq = mlnx_pre_create_cq;
p_uvp->post_create_cq = mlnx_post_create_cq;
- p_uvp->pre_query_cq = NULL;
+ p_uvp->pre_query_cq = mlnx_pre_query_cq;
p_uvp->post_query_cq = NULL;
p_uvp->pre_resize_cq = mlnx_pre_resize_cq;
@@ -434,26 +434,18 @@
ib_api_status_t
mlnx_pre_query_cq (
- IN const ib_cq_handle_t h_uvp_cq,
- IN OUT ci_umv_buf_t *p_umv_buf)
+ IN const ib_cq_handle_t h_uvp_cq,
+ OUT uint32_t* const p_size,
+ IN OUT ci_umv_buf_t *p_umv_buf)
{
- FUNC_ENTER;
- CL_ASSERT(p_umv_buf);
- p_umv_buf->command = TRUE;
- FUNC_EXIT;
- return IB_SUCCESS;
-}
+ mlnx_ual_cq_info_t *p_cq_info = (mlnx_ual_cq_info_t *)((void*)
h_uvp_cq);
+ FUNC_ENTER;
-void
-mlnx_post_query_cq (
- IN const ib_cq_handle_t h_uvp_cq,
- IN ib_api_status_t ioctl_status,
- IN OUT ci_umv_buf_t *p_umv_buf)
-{
- FUNC_ENTER;
- FUNC_EXIT;
- return;
+ *p_size = p_cq_info->cq_size;
+
+ FUNC_EXIT;
+ return IB_VERBS_PROCESSING_DONE;
}
Index: hw/mt23108/user/mlnx_ual_main.h
===================================================================
--- hw/mt23108/user/mlnx_ual_main.h (revision 61)
+++ hw/mt23108/user/mlnx_ual_main.h (working copy)
@@ -250,15 +250,10 @@
ib_api_status_t
mlnx_pre_query_cq (
- IN const ib_cq_handle_t h_uvp_cq,
- IN OUT ci_umv_buf_t *p_umv_buf);
+ IN const ib_cq_handle_t h_uvp_cq,
+ OUT uint32_t* const p_size,
+ IN OUT ci_umv_buf_t *p_umv_buf);
-void
-mlnx_post_query_cq (
- IN const ib_cq_handle_t h_uvp_cq,
- IN ib_api_status_t ioctl_status,
- IN OUT ci_umv_buf_t *p_umv_buf);
-
ib_api_status_t
mlnx_pre_destroy_cq (
IN const ib_cq_handle_t h_uvp_cq);
-------------- next part --------------
A non-text attachment was scrubbed...
Name: query_cq.patch
Type: application/octet-stream
Size: 6598 bytes
Desc: not available
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20050908/18979c6e/attachment.obj>
More information about the ofw
mailing list