[ofa-general] [RFC/PATCH] mthca: ensure alignment of doorbell writes
akepner at sgi.com
akepner at sgi.com
Wed Jul 25 18:49:31 PDT 2007
On ia64 we sometimes get "kernel unaligned access"
exceptions when doing doorbell writes. How about
something like the following to fix things up?
Tested on ia64 with a Mellanox MT23108 HCA.
mthca_cq.c | 33 +++++++++++++++------------------
mthca_doorbell.h | 15 ++++++++++-----
mthca_eq.c | 28 ++++++++++++----------------
mthca_qp.c | 41 ++++++++++++++++++-----------------------
mthca_srq.c | 16 +++++++---------
5 files changed, 62 insertions(+), 71 deletions(-)
Signed-off-by: Arthur Kepner <akepner at sgi.com>
--
diff -rup ofa_kernel-1.2.orig/drivers/infiniband/hw/mthca/mthca_cq.c ofa_kernel-1.2/drivers/infiniband/hw/mthca/mthca_cq.c
--- ofa_kernel-1.2.orig/drivers/infiniband/hw/mthca/mthca_cq.c 2007-07-20 14:42:52.858494231 -0700
+++ ofa_kernel-1.2/drivers/infiniband/hw/mthca/mthca_cq.c 2007-07-25 17:25:16.697025633 -0700
@@ -203,17 +203,16 @@ static void dump_cqe(struct mthca_dev *d
static inline void update_cons_index(struct mthca_dev *dev, struct mthca_cq *cq,
int incr)
{
- __be32 doorbell[2];
+ union mthca_doorbell db;
if (mthca_is_memfree(dev)) {
*cq->set_ci_db = cpu_to_be32(cq->cons_index);
wmb();
} else {
- doorbell[0] = cpu_to_be32(MTHCA_TAVOR_CQ_DB_INC_CI | cq->cqn);
- doorbell[1] = cpu_to_be32(incr - 1);
+ db.val32[0] = cpu_to_be32(MTHCA_TAVOR_CQ_DB_INC_CI | cq->cqn);
+ db.val32[1] = cpu_to_be32(incr - 1);
- mthca_write64(doorbell,
- dev->kar + MTHCA_CQ_DOORBELL,
+ mthca_ring_db(db, dev->kar + MTHCA_CQ_DOORBELL,
MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
/*
* Make sure doorbells don't leak out of CQ spinlock
@@ -728,16 +727,15 @@ repoll:
int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify notify)
{
- __be32 doorbell[2];
+ union mthca_doorbell db;
- doorbell[0] = cpu_to_be32((notify == IB_CQ_SOLICITED ?
+ db.val32[0] = cpu_to_be32((notify == IB_CQ_SOLICITED ?
MTHCA_TAVOR_CQ_DB_REQ_NOT_SOL :
MTHCA_TAVOR_CQ_DB_REQ_NOT) |
to_mcq(cq)->cqn);
- doorbell[1] = (__force __be32) 0xffffffff;
+ db.val32[1] = (__force __be32) 0xffffffff;
- mthca_write64(doorbell,
- to_mdev(cq->device)->kar + MTHCA_CQ_DOORBELL,
+ mthca_ring_db(db, to_mdev(cq->device)->kar + MTHCA_CQ_DOORBELL,
MTHCA_GET_DOORBELL_LOCK(&to_mdev(cq->device)->doorbell_lock));
return 0;
@@ -746,18 +744,18 @@ int mthca_tavor_arm_cq(struct ib_cq *cq,
int mthca_arbel_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify notify)
{
struct mthca_cq *cq = to_mcq(ibcq);
- __be32 doorbell[2];
+ union mthca_doorbell db;
u32 sn;
__be32 ci;
sn = cq->arm_sn & 3;
ci = cpu_to_be32(cq->cons_index);
- doorbell[0] = ci;
- doorbell[1] = cpu_to_be32((cq->cqn << 8) | (2 << 5) | (sn << 3) |
+ db.val32[0] = ci;
+ db.val32[1] = cpu_to_be32((cq->cqn << 8) | (2 << 5) | (sn << 3) |
(notify == IB_CQ_SOLICITED ? 1 : 2));
- mthca_write_db_rec(doorbell, cq->arm_db);
+ mthca_write_db_rec(db.val32, cq->arm_db);
/*
* Make sure that the doorbell record in host memory is
@@ -765,15 +763,14 @@ int mthca_arbel_arm_cq(struct ib_cq *ibc
*/
wmb();
- doorbell[0] = cpu_to_be32((sn << 28) |
+ db.val32[0] = cpu_to_be32((sn << 28) |
(notify == IB_CQ_SOLICITED ?
MTHCA_ARBEL_CQ_DB_REQ_NOT_SOL :
MTHCA_ARBEL_CQ_DB_REQ_NOT) |
cq->cqn);
- doorbell[1] = ci;
+ db.val32[1] = ci;
- mthca_write64(doorbell,
- to_mdev(ibcq->device)->kar + MTHCA_CQ_DOORBELL,
+ mthca_ring_db(db, to_mdev(ibcq->device)->kar + MTHCA_CQ_DOORBELL,
MTHCA_GET_DOORBELL_LOCK(&to_mdev(ibcq->device)->doorbell_lock));
return 0;
diff -rup ofa_kernel-1.2.orig/drivers/infiniband/hw/mthca/mthca_doorbell.h ofa_kernel-1.2/drivers/infiniband/hw/mthca/mthca_doorbell.h
--- ofa_kernel-1.2.orig/drivers/infiniband/hw/mthca/mthca_doorbell.h 2007-07-20 14:42:52.858494231 -0700
+++ ofa_kernel-1.2/drivers/infiniband/hw/mthca/mthca_doorbell.h 2007-07-25 18:03:02.088946003 -0700
@@ -42,6 +42,11 @@
#define MTHCA_CQ_DOORBELL 0x20
#define MTHCA_EQ_DOORBELL 0x28
+union mthca_doorbell {
+ __be64 val64;
+ __be32 val32[2];
+} __attribute__ ((aligned (sizeof(__be64))));
+
#if BITS_PER_LONG == 64
/*
* Assume that we can just write a 64-bit doorbell atomically. s390
@@ -58,10 +63,10 @@ static inline void mthca_write64_raw(__b
__raw_writeq((__force u64) val, dest);
}
-static inline void mthca_write64(__be32 val[2], void __iomem *dest,
+static inline void mthca_ring_db(union mthca_doorbell db, void __iomem *dest,
spinlock_t *doorbell_lock)
{
- __raw_writeq(*(u64 *) val, dest);
+ __raw_writeq((u64)db.val64, dest);
}
static inline void mthca_write_db_rec(__be32 val[2], __be32 *db)
@@ -87,14 +92,14 @@ static inline void mthca_write64_raw(__b
__raw_writel(((__force u32 *) &val)[1], dest + 4);
}
-static inline void mthca_write64(__be32 val[2], void __iomem *dest,
+static inline void mthca_ring_db(union mthca_doorbell db, void __iomem *dest,
spinlock_t *doorbell_lock)
{
unsigned long flags;
spin_lock_irqsave(doorbell_lock, flags);
- __raw_writel((__force u32) val[0], dest);
- __raw_writel((__force u32) val[1], dest + 4);
+ __raw_writel((__force u32) db.val32[0], dest);
+ __raw_writel((__force u32) db.val32[1], dest + 4);
spin_unlock_irqrestore(doorbell_lock, flags);
}
diff -rup ofa_kernel-1.2.orig/drivers/infiniband/hw/mthca/mthca_eq.c ofa_kernel-1.2/drivers/infiniband/hw/mthca/mthca_eq.c
--- ofa_kernel-1.2.orig/drivers/infiniband/hw/mthca/mthca_eq.c 2007-07-20 14:42:52.858494231 -0700
+++ ofa_kernel-1.2/drivers/infiniband/hw/mthca/mthca_eq.c 2007-07-25 17:25:34.397279816 -0700
@@ -173,10 +173,10 @@ static inline u64 async_mask(struct mthc
static inline void tavor_set_eq_ci(struct mthca_dev *dev, struct mthca_eq *eq, u32 ci)
{
- __be32 doorbell[2];
+ union mthca_doorbell db;
- doorbell[0] = cpu_to_be32(MTHCA_EQ_DB_SET_CI | eq->eqn);
- doorbell[1] = cpu_to_be32(ci & (eq->nent - 1));
+ db.val32[0] = cpu_to_be32(MTHCA_EQ_DB_SET_CI | eq->eqn);
+ db.val32[1] = cpu_to_be32(ci & (eq->nent - 1));
/*
* This barrier makes sure that all updates to ownership bits
@@ -187,8 +187,7 @@ static inline void tavor_set_eq_ci(struc
* having set_eqe_hw() overwrite the owner field.
*/
wmb();
- mthca_write64(doorbell,
- dev->kar + MTHCA_EQ_DOORBELL,
+ mthca_ring_db(db, dev->kar + MTHCA_EQ_DOORBELL,
MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
}
@@ -212,13 +211,11 @@ static inline void set_eq_ci(struct mthc
static inline void tavor_eq_req_not(struct mthca_dev *dev, int eqn)
{
- __be32 doorbell[2];
+ union mthca_doorbell db;
- doorbell[0] = cpu_to_be32(MTHCA_EQ_DB_REQ_NOT | eqn);
- doorbell[1] = 0;
-
- mthca_write64(doorbell,
- dev->kar + MTHCA_EQ_DOORBELL,
+ db.val32[0] = cpu_to_be32(MTHCA_EQ_DB_REQ_NOT | eqn);
+ db.val32[1] = 0;
+ mthca_ring_db(db, dev->kar + MTHCA_EQ_DOORBELL,
MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
}
@@ -230,13 +227,12 @@ static inline void arbel_eq_req_not(stru
static inline void disarm_cq(struct mthca_dev *dev, int eqn, int cqn)
{
if (!mthca_is_memfree(dev)) {
- __be32 doorbell[2];
+ union mthca_doorbell db;
- doorbell[0] = cpu_to_be32(MTHCA_EQ_DB_DISARM_CQ | eqn);
- doorbell[1] = cpu_to_be32(cqn);
+ db.val32[0] = cpu_to_be32(MTHCA_EQ_DB_DISARM_CQ | eqn);
+ db.val32[1] = cpu_to_be32(cqn);
- mthca_write64(doorbell,
- dev->kar + MTHCA_EQ_DOORBELL,
+ mthca_ring_db(db, dev->kar + MTHCA_EQ_DOORBELL,
MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
}
}
diff -rup ofa_kernel-1.2.orig/drivers/infiniband/hw/mthca/mthca_qp.c ofa_kernel-1.2/drivers/infiniband/hw/mthca/mthca_qp.c
--- ofa_kernel-1.2.orig/drivers/infiniband/hw/mthca/mthca_qp.c 2007-07-20 14:42:52.858494231 -0700
+++ ofa_kernel-1.2/drivers/infiniband/hw/mthca/mthca_qp.c 2007-07-25 17:25:58.057619693 -0700
@@ -1730,16 +1730,15 @@ int mthca_tavor_post_send(struct ib_qp *
out:
if (likely(nreq)) {
- __be32 doorbell[2];
+ union mthca_doorbell db;
- doorbell[0] = cpu_to_be32(((qp->sq.next_ind << qp->sq.wqe_shift) +
+ db.val32[0] = cpu_to_be32(((qp->sq.next_ind << qp->sq.wqe_shift) +
qp->send_wqe_offset) | f0 | op0);
- doorbell[1] = cpu_to_be32((qp->qpn << 8) | size0);
+ db.val32[1] = cpu_to_be32((qp->qpn << 8) | size0);
wmb();
- mthca_write64(doorbell,
- dev->kar + MTHCA_SEND_DOORBELL,
+ mthca_ring_db(db, dev->kar + MTHCA_SEND_DOORBELL,
MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
/*
* Make sure doorbells don't leak out of SQ spinlock
@@ -1760,7 +1759,7 @@ int mthca_tavor_post_receive(struct ib_q
{
struct mthca_dev *dev = to_mdev(ibqp->device);
struct mthca_qp *qp = to_mqp(ibqp);
- __be32 doorbell[2];
+ union mthca_doorbell db;
unsigned long flags;
int err = 0;
int nreq;
@@ -1836,13 +1835,12 @@ int mthca_tavor_post_receive(struct ib_q
if (unlikely(nreq == MTHCA_TAVOR_MAX_WQES_PER_RECV_DB)) {
nreq = 0;
- doorbell[0] = cpu_to_be32((qp->rq.next_ind << qp->rq.wqe_shift) | size0);
- doorbell[1] = cpu_to_be32(qp->qpn << 8);
+ db.val32[0] = cpu_to_be32((qp->rq.next_ind << qp->rq.wqe_shift) | size0);
+ db.val32[1] = cpu_to_be32(qp->qpn << 8);
wmb();
- mthca_write64(doorbell,
- dev->kar + MTHCA_RECEIVE_DOORBELL,
+ mthca_ring_db(db, dev->kar + MTHCA_RECEIVE_DOORBELL,
MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
qp->rq.head += MTHCA_TAVOR_MAX_WQES_PER_RECV_DB;
@@ -1852,13 +1850,12 @@ int mthca_tavor_post_receive(struct ib_q
out:
if (likely(nreq)) {
- doorbell[0] = cpu_to_be32((qp->rq.next_ind << qp->rq.wqe_shift) | size0);
- doorbell[1] = cpu_to_be32((qp->qpn << 8) | nreq);
+ db.val32[0] = cpu_to_be32((qp->rq.next_ind << qp->rq.wqe_shift) | size0);
+ db.val32[1] = cpu_to_be32((qp->qpn << 8) | nreq);
wmb();
- mthca_write64(doorbell,
- dev->kar + MTHCA_RECEIVE_DOORBELL,
+ mthca_ring_db(db, dev->kar + MTHCA_RECEIVE_DOORBELL,
MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
}
@@ -1880,7 +1877,7 @@ int mthca_arbel_post_send(struct ib_qp *
{
struct mthca_dev *dev = to_mdev(ibqp->device);
struct mthca_qp *qp = to_mqp(ibqp);
- __be32 doorbell[2];
+ union mthca_doorbell db;
void *wqe;
void *prev_wqe;
unsigned long flags;
@@ -1903,10 +1900,10 @@ int mthca_arbel_post_send(struct ib_qp *
if (unlikely(nreq == MTHCA_ARBEL_MAX_WQES_PER_SEND_DB)) {
nreq = 0;
- doorbell[0] = cpu_to_be32((MTHCA_ARBEL_MAX_WQES_PER_SEND_DB << 24) |
+ db.val32[0] = cpu_to_be32((MTHCA_ARBEL_MAX_WQES_PER_SEND_DB << 24) |
((qp->sq.head & 0xffff) << 8) |
f0 | op0);
- doorbell[1] = cpu_to_be32((qp->qpn << 8) | size0);
+ db.val32[1] = cpu_to_be32((qp->qpn << 8) | size0);
qp->sq.head += MTHCA_ARBEL_MAX_WQES_PER_SEND_DB;
size0 = 0;
@@ -1923,8 +1920,7 @@ int mthca_arbel_post_send(struct ib_qp *
* write MMIO send doorbell.
*/
wmb();
- mthca_write64(doorbell,
- dev->kar + MTHCA_SEND_DOORBELL,
+ mthca_ring_db(db, dev->kar + MTHCA_SEND_DOORBELL,
MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
}
@@ -2108,10 +2104,10 @@ int mthca_arbel_post_send(struct ib_qp *
out:
if (likely(nreq)) {
- doorbell[0] = cpu_to_be32((nreq << 24) |
+ db.val32[0] = cpu_to_be32((nreq << 24) |
((qp->sq.head & 0xffff) << 8) |
f0 | op0);
- doorbell[1] = cpu_to_be32((qp->qpn << 8) | size0);
+ db.val32[1] = cpu_to_be32((qp->qpn << 8) | size0);
qp->sq.head += nreq;
@@ -2127,8 +2123,7 @@ out:
* write MMIO send doorbell.
*/
wmb();
- mthca_write64(doorbell,
- dev->kar + MTHCA_SEND_DOORBELL,
+ mthca_ring_db(db, dev->kar + MTHCA_SEND_DOORBELL,
MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
}
diff -rup ofa_kernel-1.2.orig/drivers/infiniband/hw/mthca/mthca_srq.c ofa_kernel-1.2/drivers/infiniband/hw/mthca/mthca_srq.c
--- ofa_kernel-1.2.orig/drivers/infiniband/hw/mthca/mthca_srq.c 2007-07-20 14:42:52.862494291 -0700
+++ ofa_kernel-1.2/drivers/infiniband/hw/mthca/mthca_srq.c 2007-07-25 17:26:07.925761483 -0700
@@ -485,7 +485,7 @@ int mthca_tavor_post_srq_recv(struct ib_
{
struct mthca_dev *dev = to_mdev(ibsrq->device);
struct mthca_srq *srq = to_msrq(ibsrq);
- __be32 doorbell[2];
+ union mthca_doorbell db;
unsigned long flags;
int err = 0;
int first_ind;
@@ -565,8 +565,8 @@ int mthca_tavor_post_srq_recv(struct ib_
if (unlikely(nreq == MTHCA_TAVOR_MAX_WQES_PER_RECV_DB)) {
nreq = 0;
- doorbell[0] = cpu_to_be32(first_ind << srq->wqe_shift);
- doorbell[1] = cpu_to_be32(srq->srqn << 8);
+ db.val32[0] = cpu_to_be32(first_ind << srq->wqe_shift);
+ db.val32[1] = cpu_to_be32(srq->srqn << 8);
/*
* Make sure that descriptors are written
@@ -574,8 +574,7 @@ int mthca_tavor_post_srq_recv(struct ib_
*/
wmb();
- mthca_write64(doorbell,
- dev->kar + MTHCA_RECEIVE_DOORBELL,
+ mthca_ring_db(db, dev->kar + MTHCA_RECEIVE_DOORBELL,
MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
first_ind = srq->first_free;
@@ -583,8 +582,8 @@ int mthca_tavor_post_srq_recv(struct ib_
}
if (likely(nreq)) {
- doorbell[0] = cpu_to_be32(first_ind << srq->wqe_shift);
- doorbell[1] = cpu_to_be32((srq->srqn << 8) | nreq);
+ db.val32[0] = cpu_to_be32(first_ind << srq->wqe_shift);
+ db.val32[1] = cpu_to_be32((srq->srqn << 8) | nreq);
/*
* Make sure that descriptors are written before
@@ -592,8 +591,7 @@ int mthca_tavor_post_srq_recv(struct ib_
*/
wmb();
- mthca_write64(doorbell,
- dev->kar + MTHCA_RECEIVE_DOORBELL,
+ mthca_ring_db(db, dev->kar + MTHCA_RECEIVE_DOORBELL,
MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
}
--
Arthur
More information about the general
mailing list