[ofw] patch: Add code that replaces interrupts with a thread that is polling
Tzachi Dar
tzachid at mellanox.co.il
Mon Dec 22 13:13:51 PST 2008
In one of our clients systems we have noticed that during shutdown it is
possibale that an interrupt will be lost.
This patch allows the user to activate the thread and continue the
shutdown.
Thanks
Tzachi
Index: mlx4/kernel/bus/ib/main.c
===================================================================
--- mlx4/kernel/bus/ib/main.c (revision 1763)
+++ mlx4/kernel/bus/ib/main.c (working copy)
@@ -38,6 +38,11 @@
#include "ib_cache.h"
#include "net\mlx4.h"
+#if WORKAROUND_POLL_EQ
+void mlx4_poll_eq(struct ib_device *dev, BOOLEAN bStart);
+#endif
+
+
static void init_query_mad(struct ib_smp *mad)
{
mad->base_version = 1;
@@ -564,7 +569,9 @@
ibdev->ib_dev.x.get_cached_pkey = ib_get_cached_pkey;
ibdev->ib_dev.x.register_ev_cb = mlx4_reset_cb_register;
ibdev->ib_dev.x.unregister_ev_cb = mlx4_reset_cb_unregister;
-
+#if WORKAROUND_POLL_EQ
+ ibdev->ib_dev.x.poll_eq = mlx4_poll_eq;
+#endif
if (mlx4_is_livefish(ibdev->dev))
return ibdev;
@@ -674,3 +681,4 @@
}
+
Index: mlx4/kernel/bus/inc/ib_verbs_ex.h
===================================================================
--- mlx4/kernel/bus/inc/ib_verbs_ex.h (revision 1763)
+++ mlx4/kernel/bus/inc/ib_verbs_ex.h (working copy)
@@ -52,6 +52,9 @@
u8 port_num, __be16 pkey, u16 *index);
int (*register_ev_cb) (struct ib_event_handler *event_handler);
int (*unregister_ev_cb) (struct ib_event_handler *event_handler);
+#if 1 //WORKAROUND_POLL_EQ
+ void (*poll_eq)(struct ib_device *device, BOOLEAN bStart);
+#endif
};
@@ -89,3 +92,4 @@
};
+
Index: mlx4/kernel/bus/net/eq.c
===================================================================
--- mlx4/kernel/bus/net/eq.c (revision 1763)
+++ mlx4/kernel/bus/net/eq.c (working copy)
@@ -255,6 +255,103 @@
return (BOOLEAN)work;
}
+#if WORKAROUND_POLL_EQ
+
+BOOLEAN
+ IsrSynchronizeRoutine(
+ IN PVOID SynchronizeContext
+ )
+{
+ struct mlx4_dev *dev = (struct mlx4_dev *)SynchronizeContext;
+
+ mlx4_interrupt(dev->pdev->int_obj,dev);
+
+ return TRUE;
+}
+
+
+VOID eq_polling_thread(void *ctx)
+{
+#define POLLING_INTERVAL_MS 50
+ NTSTATUS status;
+ struct mlx4_priv *priv = (struct mlx4_priv *)ctx;
+ PVOID wait_objects[2];
+ LARGE_INTEGER wait_time;
+
+ wait_objects[0] = &priv->eq_table.thread_stop_event;
+ wait_objects[1] = &priv->eq_table.thread_start_event;
+
+ for(;;){
+
+ /* before start polling */
+ DbgPrint("Before polling.\n");
+ for (;;) {
+ status = KeWaitForMultipleObjects( 2, wait_objects,
+ WaitAny, Executive, KernelMode, FALSE, NULL, NULL );
+
+ if ( status == STATUS_WAIT_0 ){/* thread stopped */
+ //KeClearEvent(&priv->eq_table.thread_stop_event);
+ DbgPrint("Signaled to stop polling.\n");
+ break;
+ }
+
+ /* start polling */
+ if ( status == STATUS_WAIT_1 ){
+ //KeClearEvent(&priv->eq_table.thread_start_event);
+ DbgPrint("Signaled to start polling.\n");
+ break;
+ }
+
+ }
+
+ if(priv->eq_table.bTerminated) break;
+ if ( status == STATUS_WAIT_0 ) continue;/* thread stopped, wait for
start again */
+
+ /* polling */
+ DbgPrint("Start polling.\n");
+ wait_time.QuadPart = -(int64_t)(((uint64_t)POLLING_INTERVAL_MS) *
10000);
+ for (;;) {
+ //mlx4_interrupt( NULL, &priv->dev );
+ KeSynchronizeExecution(priv->dev.pdev->int_obj,
IsrSynchronizeRoutine, &priv->dev);
+
+ status = KeWaitForSingleObject( &priv->eq_table.thread_stop_event,
+ Executive, KernelMode, FALSE, &wait_time );
+ if ( status == STATUS_SUCCESS ) {
+ //KeClearEvent(&priv->eq_table.thread_stop_event);
+ DbgPrint("Signaled to stop polling while in polling mode.\n");
+ break; /* thread stopped */
+ }
+ }
+
+ if(priv->eq_table.bTerminated) break;
+ }
+
+ DbgPrint("Polling thread terminated.\n");
+ PsTerminateSystemThread(STATUS_SUCCESS);
+
+}
+
+
+void mlx4_poll_eq(struct ib_device *device, BOOLEAN bStart)
+{
+ LONG signalled=0;
+ struct mlx4_priv *priv = mlx4_priv(device->dma_device);
+
+ if(bStart){
+ /* signal start of polling */
+ signalled = KeSetEvent(
+ &priv->eq_table.thread_start_event, IO_NO_INCREMENT, FALSE );
+ }else{
+ /* signal end of polling */
+ signalled = KeSetEvent(
+ &priv->eq_table.thread_stop_event, IO_NO_INCREMENT, FALSE );
+ }
+
+}
+
+#endif
+
+
#ifdef CONFIG_PCI_MSI
/* not ported yet */
@@ -548,6 +645,8 @@
__free_page(dev->pdev, priv->eq_table.icm_page);
}
+
+
int mlx4_init_eq_table(struct mlx4_dev *dev)
{
struct mlx4_priv *priv = mlx4_priv(dev);
@@ -648,6 +747,42 @@
for (i = 0; i < MLX4_NUM_EQ; ++i)
eq_set_ci(&priv->eq_table.eq[i], 1);
+
+#if WORKAROUND_POLL_EQ
+ { /* Create a thread for polling EQs in case of missing interrupts
from the card */
+ NTSTATUS status;
+ OBJECT_ATTRIBUTES attr;
+ HANDLE handle;
+
+ KeInitializeEvent(&priv->eq_table.thread_start_event,
SynchronizationEvent, FALSE);
+ KeInitializeEvent(&priv->eq_table.thread_stop_event,
SynchronizationEvent, FALSE);
+ //KeInitializeEvent(&priv->eq_table.thread_start_event,
NotificationEvent , FALSE);
+ //KeInitializeEvent(&priv->eq_table.thread_stop_event,
NotificationEvent , FALSE);
+ priv->eq_table.bTerminated = FALSE;
+ priv->eq_table.threadObject = NULL;
+ InitializeObjectAttributes( &attr, NULL, OBJ_KERNEL_HANDLE, NULL,
NULL );
+ status = PsCreateSystemThread( &handle,
+ THREAD_ALL_ACCESS, &attr, NULL, NULL, eq_polling_thread, priv );
+ if (NT_SUCCESS(status)) {
+ status = ObReferenceObjectByHandle(
+ handle,
+ THREAD_ALL_ACCESS,
+ NULL,
+ KernelMode,
+ &priv->eq_table.threadObject,
+ NULL
+ );
+
+ ASSERT(status == STATUS_SUCCESS); //
+
+ status = ZwClose(handle);
+
+ ASSERT(NT_SUCCESS(status)); // Should always succeed
+
+ }
+ }
+#endif
+
return 0;
#ifdef USE_WDM_INTERRUPTS
@@ -737,6 +872,33 @@
struct mlx4_priv *priv = mlx4_priv(dev);
int i;
+#if WORKAROUND_POLL_EQ
+ /* stop the EQ polling thread */
+ if (priv->eq_table.threadObject) {
+ #define WAIT_TIME_MS 3000
+ NTSTATUS status;
+ LARGE_INTEGER wait_time;
+ LONG signalled;
+
+ priv->eq_table.bTerminated = TRUE;
+
+ /* signal polling stopped in case it is not */
+ signalled = KeSetEvent(
+ &priv->eq_table.thread_stop_event, IO_NO_INCREMENT, FALSE );
+
+ /* wait for completion of the thread */
+ wait_time.QuadPart = -(int64_t)(((uint64_t)WAIT_TIME_MS) * 10000);
+ status = KeWaitForSingleObject( priv->eq_table.threadObject,
+ Executive, KernelMode, FALSE, &wait_time );
+ ASSERT(status == STATUS_SUCCESS);
+
+ ObDereferenceObject(priv->eq_table.threadObject);
+
+ /* cleanup */
+ priv->eq_table.threadObject = NULL;
+ }
+#endif
+
mlx4_MAP_EQ(dev, MLX4_ASYNC_EVENT_MASK, 1,
priv->eq_table.eq[MLX4_EQ_ASYNC].eqn);
Index: mlx4/kernel/bus/net/mlx4.h
===================================================================
--- mlx4/kernel/bus/net/mlx4.h (revision 1763)
+++ mlx4/kernel/bus/net/mlx4.h (working copy)
@@ -233,6 +233,12 @@
int have_irq;
u8 inta_pin;
u8 max_extra_eqs;
+#if WORKAROUND_POLL_EQ
+ KEVENT thread_start_event;
+ KEVENT thread_stop_event;
+ BOOLEAN bTerminated;
+ PVOID threadObject;
+#endif
};
struct mlx4_srq_table {
Index: mlx4/kernel/hca/fw.c
===================================================================
--- mlx4/kernel/hca/fw.c (revision 1763)
+++ mlx4/kernel/hca/fw.c (working copy)
@@ -404,8 +404,24 @@
UNREFERENCED_PARAMETER(handle_array);
UNREFERENCED_PARAMETER(num_handles);
- if( !p_umv_buf )
- return IB_UNSUPPORTED;
+ if(!p_umv_buf ) {
+ #if 1 //WORKAROUND_POLL_EQ
+ if ((p_ci_op->command == FW_POLL_EQ_START) || (p_ci_op->command
== FW_POLL_EQ_STOP)){ // poll EQ (in case of missed interrupt)
+ mlnx_hca_t *p_hca = (mlnx_hca_t *)h_ca;
+ struct ib_device *p_ibdev = hca2ibdev(p_hca);
+ if(p_ci_op->command == FW_POLL_EQ_START)
+ {
+ p_ibdev->x.poll_eq(p_ibdev,1);
+ }else
+ {
+ p_ibdev->x.poll_eq(p_ibdev,0);
+ }
+ return IB_SUCCESS;
+ }
+ else
+ #endif
+ return IB_UNSUPPORTED;
+ }
if ( !p_ci_op )
return IB_INVALID_PARAMETER;
@@ -475,3 +491,4 @@
}
}
+
Index: mlx4/kernel/inc/vc.h
===================================================================
--- mlx4/kernel/inc/vc.h (revision 1763)
+++ mlx4/kernel/inc/vc.h (working copy)
@@ -51,6 +51,11 @@
#define FW_OPEN_IF 0xe7
#define FW_CLOSE_IF 0x7e
+#if WORKAROUND_POLL_EQ
+#define FW_POLL_EQ_START 0x0D
+#define FW_POLL_EQ_STOP 0x0E
+#endif
+
/* uplink info */
typedef struct {
uint8_t bus_type; /* 1 - PCI, 2 - PCI-X, 3 - PCI_E */
@@ -74,3 +79,4 @@
} uplink_info_t;
+
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20081222/8dede475/attachment.html>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: mlx4_poll_eq.patch
Type: application/octet-stream
Size: 8480 bytes
Desc: mlx4_poll_eq.patch
URL: <http://lists.openfabrics.org/pipermail/ofw/attachments/20081222/8dede475/attachment.obj>
More information about the ofw
mailing list