[openib-general] [PATCH] ib_mad: PCI mapping/unmapping and some buffering fixes
Hal Rosenstock
halr at voltaire.com
Wed Sep 15 06:41:37 PDT 2004
ib_mad: PCI mapping/unmapping and some buffering fixes
Index: access/ib_mad_priv.h
===================================================================
--- access/ib_mad_priv.h (revision 837)
+++ access/ib_mad_priv.h (working copy)
@@ -106,8 +106,10 @@
struct ib_mad_send_wr_private {
struct list_head send_list;
struct ib_mad_agent *agent;
- u64 wr_id;
+ u64 wr_id; /* client WRID */
int timeout_ms;
+ struct ib_mad_buf *buf;
+ u32 buf_len;
};
struct ib_mad_mgmt_method_table {
Index: access/ib_mad.c
===================================================================
--- access/ib_mad.c (revision 837)
+++ access/ib_mad.c (working copy)
@@ -81,6 +81,8 @@
struct ib_mad_agent_private *priv);
static void remove_mad_reg_req(struct ib_mad_agent_private *priv);
static int ib_mad_port_restart(struct ib_mad_port_private *priv);
+static int ib_mad_post_receive_mad(struct ib_mad_port_private
*port_priv,
+ struct ib_qp *qp);
static int ib_mad_post_receive_mads(struct ib_mad_port_private *priv);
static inline u8 convert_mgmt_class(u8 mgmt_class);
@@ -291,6 +293,7 @@
struct ib_send_wr wr;
struct ib_send_wr *bad_wr;
struct ib_mad_send_wr_private *mad_send_wr;
+ struct ib_sge gather_list;
unsigned long flags;
cur_send_wr = send_wr;
@@ -300,7 +303,7 @@
return -EINVAL;
}
- /* Walk list of send WRs and post each one on send list */
+ /* Walk list of send WRs and post each on send list */
cur_send_wr = send_wr;
while (cur_send_wr) {
next_send_wr = (struct ib_send_wr *)cur_send_wr->next;
@@ -314,16 +317,26 @@
printk(KERN_ERR "No memory for ib_mad_send_wr_private\n");
return -ENOMEM;
}
+
+ /* Setup gather list */
+ gather_list.addr = pci_map_single(mad_agent->device->dma_device,
+ cur_send_wr->sg_list->addr,
+ cur_send_wr->sg_list->length,
+ PCI_DMA_TODEVICE);
+ gather_list.length = cur_send_wr->sg_list->length;
+ gather_list.lkey = cur_send_wr->sg_list->lkey;
+
/* Initialize MAD send WR tracking structure */
mad_send_wr->agent = mad_agent;
mad_send_wr->wr_id = cur_send_wr->wr_id;
/* Timeout valid only when MAD is a request !!! */
mad_send_wr->timeout_ms = cur_send_wr->wr.ud.timeout_ms;
+ mad_send_wr->buf_len = gather_list.length;
wr.next = NULL;
wr.opcode = IB_WR_SEND; /* cur_send_wr->opcode ? */
wr.wr_id = (unsigned long)mad_send_wr;
- wr.sg_list = cur_send_wr->sg_list;
+ wr.sg_list = &gather_list;
wr.num_sge = cur_send_wr->num_sge;
wr.wr.ud.remote_qpn = cur_send_wr->wr.ud.remote_qpn;
wr.wr.ud.remote_qkey = cur_send_wr->wr.ud.remote_qkey;
@@ -338,8 +351,17 @@
((struct ib_mad_port_private
*)mad_agent->device->mad)->send_posted_mad_count++;
spin_unlock_irqrestore(&((struct ib_mad_port_private
*)mad_agent->device->mad)->send_list_lock, flags);
+ pci_unmap_addr_set(&mad_send_wr->buf, mapping,
+ gather_list.addr);
+
ret = ib_post_send(mad_agent->qp, &wr, &bad_wr);
if (ret) {
+ pci_unmap_single(mad_agent->device->dma_device,
+ pci_unmap_addr(&mad_send_wr->buf,
+ mapping),
+ gather_list.length,
+ PCI_DMA_TODEVICE);
+
/* Unlink from posted send MAD list */
spin_unlock_irqrestore(&((struct ib_mad_port_private
*)mad_agent->device->mad)->send_list_lock, flags);
list_del(&mad_send_wr->send_list);
@@ -685,9 +707,12 @@
recv =
list_entry(&port_priv->recv_posted_mad_list[convert_qpnum(qp_num)],
struct ib_mad_private,
header.mad_list);
+
/* Remove from posted receive MAD list */
list_del(&recv->header.mad_list);
+
port_priv->recv_posted_mad_count[convert_qpnum(qp_num)]--;
+
} else {
printk(KERN_ERR "Receive completion WR ID 0x%Lx on QP %d with no
posted receive\n", wc->wr_id, qp_num);
spin_unlock_irqrestore(&port_priv->recv_list_lock, flags);
@@ -695,6 +720,11 @@
}
spin_unlock_irqrestore(&port_priv->recv_list_lock, flags);
+ pci_unmap_single(port_priv->device->dma_device,
+ pci_unmap_addr(&recv->buf, mapping),
+ sizeof(struct ib_mad_private) - sizeof(struct
ib_mad_private_header),
+ PCI_DMA_FROMDEVICE);
+
/* Setup MAD receive work completion from "normal" work completion */
recv_wc.wc = wc;
recv_wc.mad_len = sizeof(struct ib_mad); /* Should this be based on
wc->byte_len ? Also, RMPP !!! */
@@ -735,8 +765,8 @@
}
spin_unlock_irqrestore(&port_priv->reg_lock, flags);
- /* Repost receive request */
- /* Client must be done with receive */
+ /* Post another receive request for this QP */
+ ib_mad_post_receive_mad(port_priv, port_priv->qp[qp_num]);
ret:
return;
@@ -754,22 +784,30 @@
send_wr = list_entry(&port_priv->send_posted_mad_list,
struct ib_mad_send_wr_private,
send_list);
+
if (send_wr->wr_id != wc->wr_id) {
printk(KERN_ERR "Send completion WR ID 0x%Lx doesn't match posted
send WR ID 0x%Lx\n", wc->wr_id, send_wr->wr_id);
goto error;
}
+
/* Check whether timeout was requested !!! */
/* Remove from posted send MAD list */
list_del(&send_wr->send_list);
port_priv->send_posted_mad_count--;
+
} else {
printk(KERN_ERR "Send completion WR ID 0x%Lx but send list is
empty\n", wc->wr_id);
goto error;
}
spin_unlock_irqrestore(&port_priv->send_list_lock, flags);
+ pci_unmap_single(port_priv->device->dma_device,
+ pci_unmap_addr(&send_wr->buf, mapping),
+ send_wr->buf_len,
+ PCI_DMA_TODEVICE);
+
/* Restore client wr_id in WC */
wc->wr_id = send_wr->wr_id;
/* Invoke client send callback */
@@ -916,10 +954,18 @@
struct ib_recv_wr *bad_recv_wr;
unsigned long flags;
- /* Allocate memory for receive MAD (and private header) */
- mad_priv = kmalloc(sizeof *mad_priv, GFP_KERNEL);
+ /*
+ * Allocate memory for receive buffer.
+ * This is for both MAD and private header
+ * which serves as the receive tracking structure.
+ * By prepending thisheader, there is one rather
+ * than two memory allocations.
+ */
+ mad_priv = kmalloc(sizeof *mad_priv,
+ (in_atomic() || irqs_disabled()) ?
+ GFP_ATOMIC : GFP_KERNEL);
if (!mad_priv) {
- printk(KERN_ERR "No memory for receive MAD\n");
+ printk(KERN_ERR "No memory for receive buffer\n");
return -ENOMEM;
}
@@ -949,16 +995,18 @@
/* Now, post receive WR */
if (ib_post_recv(qp, &recv_wr, &bad_recv_wr)) {
+
+ pci_unmap_single(port_priv->device->dma_device,
+ pci_unmap_addr(&mad_priv->header.buf, mapping),
+ sizeof *mad_priv - sizeof mad_priv->header,
+ PCI_DMA_FROMDEVICE);
+
/* Unlink from posted receive MAD list */
spin_lock_irqsave(&port_priv->recv_list_lock, flags);
list_del(&mad_priv->header.mad_list);
port_priv->recv_posted_mad_count[convert_qpnum(qp->qp_num)]--;
spin_unlock_irqrestore(&port_priv->recv_list_lock, flags);
- pci_unmap_single(port_priv->device->dma_device,
- pci_unmap_addr(&mad_priv->header.buf, mapping),
- sizeof *mad_priv - sizeof mad_priv->header,
- PCI_DMA_FROMDEVICE);
kfree(mad_priv);
printk(KERN_NOTICE "ib_post_recv failed\n");
return -EINVAL;
@@ -968,14 +1016,14 @@
}
/*
- * Get receive MADs and post receive WRs for them
+ * Allocate receive MADs and post receive WRs for them
*/
static int ib_mad_post_receive_mads(struct ib_mad_port_private
*port_priv)
{
int i, j;
for (i = 0; i < IB_MAD_QP_RECV_SIZE; i++) {
- for (j = 0; j < 2; j++) {
+ for (j = 0; j < IB_MAD_QPS_CORE; j++) {
if (ib_mad_post_receive_mad(port_priv,
port_priv->qp[j])) {
printk(KERN_ERR "receive post %d failed\n", i + 1);
@@ -994,12 +1042,12 @@
int i;
unsigned long flags;
- /* PCI mapping ? */
-
for (i = 0; i < IB_MAD_QPS_SUPPORTED; i++) {
spin_lock_irqsave(&port_priv->recv_list_lock, flags);
while (!list_empty(&port_priv->recv_posted_mad_list[i])) {
+ /* PCI mapping !!! */
+
}
INIT_LIST_HEAD(&port_priv->recv_posted_mad_list[i]);
port_priv->recv_posted_mad_count[i] = 0;
@@ -1014,11 +1062,13 @@
{
unsigned long flags;
- /* PCI mapping ? */
-
spin_lock_irqsave(&port_priv->send_list_lock, flags);
while (!list_empty(&port_priv->send_posted_mad_list)) {
+
+ /* PCI mapping !!! */
+
list_del(&port_priv->send_posted_mad_list);
+
/* Call completion handler with some status ? */
}
More information about the general
mailing list