[ofa-general] [PATCH 6/6] IB/ipath - Add locking for interrupt use of ipath_pd contexts vs free.
Ralph Campbell
ralph.campbell at qlogic.com
Wed Dec 3 10:37:18 PST 2008
From: Dave Olson <dave.olson at qlogic.com>
Fixes timing race resulting in panic. Not a performance sensitive path.
Signed-off-by: Dave Olson <dave.olson at qlogic.com>
---
drivers/infiniband/hw/ipath/ipath_driver.c | 49 +++++++++++++++----------
drivers/infiniband/hw/ipath/ipath_file_ops.c | 21 +++++------
drivers/infiniband/hw/ipath/ipath_init_chip.c | 1 +
drivers/infiniband/hw/ipath/ipath_kernel.h | 2 +
drivers/infiniband/hw/ipath/ipath_keys.c | 2 +
drivers/infiniband/hw/ipath/ipath_mad.c | 2 +
drivers/infiniband/hw/ipath/ipath_verbs.c | 3 +-
7 files changed, 49 insertions(+), 31 deletions(-)
diff --git a/drivers/infiniband/hw/ipath/ipath_driver.c b/drivers/infiniband/hw/ipath/ipath_driver.c
index ad0aab6..69c0ce3 100644
--- a/drivers/infiniband/hw/ipath/ipath_driver.c
+++ b/drivers/infiniband/hw/ipath/ipath_driver.c
@@ -661,6 +661,8 @@ bail:
static void __devexit cleanup_device(struct ipath_devdata *dd)
{
int port;
+ struct ipath_portdata **tmp;
+ unsigned long flags;
if (*dd->ipath_statusp & IPATH_STATUS_CHIP_PRESENT) {
/* can't do anything more with chip; needs re-init */
@@ -742,20 +744,21 @@ static void __devexit cleanup_device(struct ipath_devdata *dd)
/*
* free any resources still in use (usually just kernel ports)
- * at unload; we do for portcnt, not cfgports, because cfgports
- * could have changed while we were loaded.
+ * at unload; we do for portcnt, because that's what we allocate.
+ * We acquire lock to be really paranoid that ipath_pd isn't being
+ * accessed from some interrupt-related code (that should not happen,
+ * but best to be sure).
*/
+ spin_lock_irqsave(&dd->ipath_uctxt_lock, flags);
+ tmp = dd->ipath_pd;
+ dd->ipath_pd = NULL;
+ spin_unlock_irqrestore(&dd->ipath_uctxt_lock, flags);
for (port = 0; port < dd->ipath_portcnt; port++) {
- struct ipath_portdata *pd = dd->ipath_pd[port];
- dd->ipath_pd[port] = NULL;
+ struct ipath_portdata *pd = tmp[port];
+ tmp[port] = NULL; /* debugging paranoia */
ipath_free_pddata(dd, pd);
}
- kfree(dd->ipath_pd);
- /*
- * debuggability, in case some cleanup path tries to use it
- * after this
- */
- dd->ipath_pd = NULL;
+ kfree(tmp);
}
static void __devexit ipath_remove_one(struct pci_dev *pdev)
@@ -2586,6 +2589,7 @@ int ipath_reset_device(int unit)
{
int ret, i;
struct ipath_devdata *dd = ipath_lookup(unit);
+ unsigned long flags;
if (!dd) {
ret = -ENODEV;
@@ -2611,18 +2615,21 @@ int ipath_reset_device(int unit)
goto bail;
}
+ spin_lock_irqsave(&dd->ipath_uctxt_lock, flags);
if (dd->ipath_pd)
for (i = 1; i < dd->ipath_cfgports; i++) {
- if (dd->ipath_pd[i] && dd->ipath_pd[i]->port_cnt) {
- ipath_dbg("unit %u port %d is in use "
- "(PID %u cmd %s), can't reset\n",
- unit, i,
- pid_nr(dd->ipath_pd[i]->port_pid),
- dd->ipath_pd[i]->port_comm);
- ret = -EBUSY;
- goto bail;
- }
+ if (!dd->ipath_pd[i] || !dd->ipath_pd[i]->port_cnt)
+ continue;
+ spin_unlock_irqrestore(&dd->ipath_uctxt_lock, flags);
+ ipath_dbg("unit %u port %d is in use "
+ "(PID %u cmd %s), can't reset\n",
+ unit, i,
+ pid_nr(dd->ipath_pd[i]->port_pid),
+ dd->ipath_pd[i]->port_comm);
+ ret = -EBUSY;
+ goto bail;
}
+ spin_unlock_irqrestore(&dd->ipath_uctxt_lock, flags);
if (dd->ipath_flags & IPATH_HAS_SEND_DMA)
teardown_sdma(dd);
@@ -2656,9 +2663,12 @@ static int ipath_signal_procs(struct ipath_devdata *dd, int sig)
{
int i, sub, any = 0;
struct pid *pid;
+ unsigned long flags;
if (!dd->ipath_pd)
return 0;
+
+ spin_lock_irqsave(&dd->ipath_uctxt_lock, flags);
for (i = 1; i < dd->ipath_cfgports; i++) {
if (!dd->ipath_pd[i] || !dd->ipath_pd[i]->port_cnt)
continue;
@@ -2682,6 +2692,7 @@ static int ipath_signal_procs(struct ipath_devdata *dd, int sig)
any++;
}
}
+ spin_unlock_irqrestore(&dd->ipath_uctxt_lock, flags);
return any;
}
diff --git a/drivers/infiniband/hw/ipath/ipath_file_ops.c b/drivers/infiniband/hw/ipath/ipath_file_ops.c
index ceab52c..239d4e8 100644
--- a/drivers/infiniband/hw/ipath/ipath_file_ops.c
+++ b/drivers/infiniband/hw/ipath/ipath_file_ops.c
@@ -2046,7 +2046,9 @@ static int ipath_close(struct inode *in, struct file *fp)
struct ipath_filedata *fd;
struct ipath_portdata *pd;
struct ipath_devdata *dd;
+ unsigned long flags;
unsigned port;
+ struct pid *pid;
ipath_cdbg(VERBOSE, "close on dev %lx, private data %p\n",
(long)in->i_rdev, fp->private_data);
@@ -2079,14 +2081,13 @@ static int ipath_close(struct inode *in, struct file *fp)
mutex_unlock(&ipath_mutex);
goto bail;
}
+ /* early; no interrupt users after this */
+ spin_lock_irqsave(&dd->ipath_uctxt_lock, flags);
port = pd->port_port;
-
- if (pd->port_hdrqfull) {
- ipath_cdbg(PROC, "%s[%u] had %u rcvhdrqfull errors "
- "during run\n", pd->port_comm, pid_nr(pd->port_pid),
- pd->port_hdrqfull);
- pd->port_hdrqfull = 0;
- }
+ dd->ipath_pd[port] = NULL;
+ pid = pd->port_pid;
+ pd->port_pid = NULL;
+ spin_unlock_irqrestore(&dd->ipath_uctxt_lock, flags);
if (pd->port_rcvwait_to || pd->port_piowait_to
|| pd->port_rcvnowait || pd->port_pionowait) {
@@ -2143,13 +2144,11 @@ static int ipath_close(struct inode *in, struct file *fp)
unlock_expected_tids(pd);
ipath_stats.sps_ports--;
ipath_cdbg(PROC, "%s[%u] closed port %u:%u\n",
- pd->port_comm, pid_nr(pd->port_pid),
+ pd->port_comm, pid_nr(pid),
dd->ipath_unit, port);
}
- put_pid(pd->port_pid);
- pd->port_pid = NULL;
- dd->ipath_pd[pd->port_port] = NULL; /* before releasing mutex */
+ put_pid(pid);
mutex_unlock(&ipath_mutex);
ipath_free_pddata(dd, pd); /* after releasing the mutex */
diff --git a/drivers/infiniband/hw/ipath/ipath_init_chip.c b/drivers/infiniband/hw/ipath/ipath_init_chip.c
index 3e5baa4..64aeefb 100644
--- a/drivers/infiniband/hw/ipath/ipath_init_chip.c
+++ b/drivers/infiniband/hw/ipath/ipath_init_chip.c
@@ -229,6 +229,7 @@ static int init_chip_first(struct ipath_devdata *dd)
spin_lock_init(&dd->ipath_kernel_tid_lock);
spin_lock_init(&dd->ipath_user_tid_lock);
spin_lock_init(&dd->ipath_sendctrl_lock);
+ spin_lock_init(&dd->ipath_uctxt_lock);
spin_lock_init(&dd->ipath_sdma_lock);
spin_lock_init(&dd->ipath_gpio_lock);
spin_lock_init(&dd->ipath_eep_st_lock);
diff --git a/drivers/infiniband/hw/ipath/ipath_kernel.h b/drivers/infiniband/hw/ipath/ipath_kernel.h
index aa84153..6ba4861 100644
--- a/drivers/infiniband/hw/ipath/ipath_kernel.h
+++ b/drivers/infiniband/hw/ipath/ipath_kernel.h
@@ -477,6 +477,8 @@ struct ipath_devdata {
spinlock_t ipath_kernel_tid_lock;
spinlock_t ipath_user_tid_lock;
spinlock_t ipath_sendctrl_lock;
+ /* around ipath_pd and (user ports) port_cnt use (intr vs free) */
+ spinlock_t ipath_uctxt_lock;
/*
* IPATH_STATUS_*,
diff --git a/drivers/infiniband/hw/ipath/ipath_keys.c b/drivers/infiniband/hw/ipath/ipath_keys.c
index 8f32b17..c0e933f 100644
--- a/drivers/infiniband/hw/ipath/ipath_keys.c
+++ b/drivers/infiniband/hw/ipath/ipath_keys.c
@@ -132,6 +132,7 @@ int ipath_lkey_ok(struct ipath_qp *qp, struct ipath_sge *isge,
* (see ipath_get_dma_mr and ipath_dma.c).
*/
if (sge->lkey == 0) {
+ /* always a kernel port, no locking needed */
struct ipath_pd *pd = to_ipd(qp->ibqp.pd);
if (pd->user) {
@@ -211,6 +212,7 @@ int ipath_rkey_ok(struct ipath_qp *qp, struct ipath_sge_state *ss,
* (see ipath_get_dma_mr and ipath_dma.c).
*/
if (rkey == 0) {
+ /* always a kernel port, no locking needed */
struct ipath_pd *pd = to_ipd(qp->ibqp.pd);
if (pd->user) {
diff --git a/drivers/infiniband/hw/ipath/ipath_mad.c b/drivers/infiniband/hw/ipath/ipath_mad.c
index be4fc9a..17a1231 100644
--- a/drivers/infiniband/hw/ipath/ipath_mad.c
+++ b/drivers/infiniband/hw/ipath/ipath_mad.c
@@ -348,6 +348,7 @@ bail:
*/
static int get_pkeys(struct ipath_devdata *dd, u16 * pkeys)
{
+ /* always a kernel port, no locking needed */
struct ipath_portdata *pd = dd->ipath_pd[0];
memcpy(pkeys, pd->port_pkeys, sizeof(pd->port_pkeys));
@@ -730,6 +731,7 @@ static int set_pkeys(struct ipath_devdata *dd, u16 *pkeys)
int i;
int changed = 0;
+ /* always a kernel port, no locking needed */
pd = dd->ipath_pd[0];
for (i = 0; i < ARRAY_SIZE(pd->port_pkeys); i++) {
diff --git a/drivers/infiniband/hw/ipath/ipath_verbs.c b/drivers/infiniband/hw/ipath/ipath_verbs.c
index eabc424..cdf0e6a 100644
--- a/drivers/infiniband/hw/ipath/ipath_verbs.c
+++ b/drivers/infiniband/hw/ipath/ipath_verbs.c
@@ -1852,7 +1852,7 @@ unsigned ipath_get_npkeys(struct ipath_devdata *dd)
}
/**
- * ipath_get_pkey - return the indexed PKEY from the port 0 PKEY table
+ * ipath_get_pkey - return the indexed PKEY from the port PKEY table
* @dd: the infinipath device
* @index: the PKEY index
*/
@@ -1860,6 +1860,7 @@ unsigned ipath_get_pkey(struct ipath_devdata *dd, unsigned index)
{
unsigned ret;
+ /* always a kernel port, no locking needed */
if (index >= ARRAY_SIZE(dd->ipath_pd[0]->port_pkeys))
ret = 0;
else
More information about the general
mailing list