[openib-general] [PATCH] ib_mad: Receive path fixes
Hal Rosenstock
halr at voltaire.com
Mon Sep 27 06:00:27 PDT 2004
ib_mad: Receive path fixes
I have now successfully handled a receive SMP without a matching
registration including the reposting of receive buffer (working around
(not included in this patch) a bug relative to the receive list
handling). I think the receive side will be working once I get past that
:-)
Index: ib_mad_priv.h
===================================================================
--- ib_mad_priv.h (revision 885)
+++ ib_mad_priv.h (working copy)
@@ -77,6 +77,14 @@
#define MAX_MGMT_VERSION 8
+union ib_mad_recv_wrid {
+ u64 wrid;
+ struct {
+ u32 index;
+ u32 qpn;
+ } wrid_field;
+};
+
struct ib_mad_private_header {
struct ib_mad_recv_wc recv_wc; /* must be first member (for now
!!!) */
struct ib_mad_recv_buf recv_buf;
@@ -117,11 +125,11 @@
struct ib_mad_thread_private {
wait_queue_head_t wait;
+ atomic_t completion_event;
};
struct ib_mad_port_private {
struct list_head port_list;
- struct task_struct *mad_thread;
struct ib_device *device;
int port_num;
struct ib_qp *qp[IB_MAD_QPS_SUPPORTED];
@@ -140,7 +148,9 @@
spinlock_t recv_list_lock;
struct list_head recv_posted_mad_list[IB_MAD_QPS_SUPPORTED];
int recv_posted_mad_count[IB_MAD_QPS_SUPPORTED];
+ u32 recv_wr_index[IB_MAD_QPS_SUPPORTED];
+ struct task_struct *mad_thread;
struct ib_mad_thread_private mad_thread_private;
};
Index: ib_mad.c
===================================================================
--- ib_mad.c (revision 885)
+++ ib_mad.c (working copy)
@@ -219,7 +219,7 @@
goto error3;
}
- /* Add mad agent into agent list */
+ /* Add mad agent into port's agent list */
list_add_tail(&mad_agent_priv->agent_list,
&port_priv->agent_list);
spin_unlock_irqrestore(&port_priv->reg_lock, flags);
@@ -257,6 +257,10 @@
&entry->agent_list, agent_list)
{
if (&entry2->agent == mad_agent) {
remove_mad_reg_req(entry2);
+
+ /* Check for any pending send MADs for
this agent !!! */
+
+ /* Remove mad agent from port's agent
list */
list_del(&entry2->agent_list);
spin_unlock_irqrestore(&entry->reg_lock,
flags2);
@@ -576,7 +580,7 @@
* as QP numbers will not be packed once redirection supported
*/
if (qp_num > 1) {
- printk(KERN_ERR "QP number %d invalid\n", qp_num);
+ return -1;
}
return qp_num;
}
@@ -680,28 +684,36 @@
struct ib_wc *wc)
{
struct ib_mad_private *recv;
+ union ib_mad_recv_wrid wrid;
unsigned long flags;
u32 qp_num;
struct ib_mad_agent_private *mad_agent;
- int solicited;
+ int solicited, qpn;
+ int callback = 0;
- /* For receive, WC WRID is the QP number */
- qp_num = wc->wr_id;
-
+ /* For receive, QP number is field in the WC WRID */
+ wrid.wrid = wc->wr_id;
+ qp_num = wrid.wrid_field.qpn;
+ qpn = convert_qpnum(qp_num);
+ if (qpn == -1) {
+ printk(KERN_ERR "Packet received on unknown QPN %d\n",
qp_num);
+ return;
+ }
+
/*
* Completion corresponds to first entry on
* posted MAD receive list based on WRID in completion
*/
spin_lock_irqsave(&port_priv->recv_list_lock, flags);
- if
(!list_empty(&port_priv->recv_posted_mad_list[convert_qpnum(qp_num)])) {
- recv =
list_entry(&port_priv->recv_posted_mad_list[convert_qpnum(qp_num)],
+ if (!list_empty(&port_priv->recv_posted_mad_list[qpn])) {
+ recv = list_entry(&port_priv->recv_posted_mad_list[qpn],
struct ib_mad_private,
header.recv_buf.list);
/* Remove from posted receive MAD list */
list_del(&recv->header.recv_buf.list);
-
port_priv->recv_posted_mad_count[convert_qpnum(qp_num)]--;
+ port_priv->recv_posted_mad_count[qpn]--;
} else {
printk(KERN_ERR "Receive completion WR ID 0x%Lx on QP %d
with no posted receive\n", wc->wr_id, qp_num);
@@ -724,7 +736,7 @@
recv->header.recv_buf.list.next = NULL; /* Until RMPP
implemented !!! */
recv->header.recv_buf.mad = (struct ib_mad *)&recv->mad;
if (wc->wc_flags & IB_WC_GRH) {
- recv->header.recv_buf.grh = (struct ib_grh *)&recv->grh;
+ recv->header.recv_buf.grh = &recv->grh;
} else {
recv->header.recv_buf.grh = NULL;
}
@@ -746,19 +758,25 @@
} else {
if (solicited) {
/* Walk the send posted list to find the match
!!! */
- printk(KERN_DEBUG "Currently unsupported
solicited MAD received\n");
+ printk(KERN_DEBUG "Receive solicited MAD
currently unsupported\n");
}
+ callback = 1;
/* Invoke receive callback */
mad_agent->agent.recv_handler(&mad_agent->agent,
&recv->header.recv_wc);
}
spin_unlock_irqrestore(&port_priv->reg_lock, flags);
+ret:
+ if (!callback) {
+ /* Should this case be optimized ? */
+ kmem_cache_free(ib_mad_cache, recv);
+ }
+
/* Post another receive request for this QP */
ib_mad_post_receive_mad(port_priv, port_priv->qp[qp_num]);
-ret:
return;
}
@@ -814,22 +832,21 @@
{
struct ib_wc wc;
int err_status = 0;
-
- while (!ib_poll_cq(port_priv->cq, 1, &wc)) {
- printk(KERN_DEBUG "Completion - WR ID = 0x%Lx\n",
wc.wr_id);
-
+
+ while (ib_poll_cq(port_priv->cq, 1, &wc) == 1) {
+ printk(KERN_DEBUG "Completion opcode 0x%x WRID 0x%Lx\n",
wc.opcode, wc.wr_id);
if (wc.status != IB_WC_SUCCESS) {
switch (wc.opcode) {
case IB_WC_SEND:
- printk(KERN_ERR "Send completion error:
%d\n",
+ printk(KERN_ERR "Send completion error
%d\n",
wc.status);
break;
case IB_WC_RECV:
- printk(KERN_ERR "Recv completion error:
%d\n",
+ printk(KERN_ERR "Recv completion error
%d\n",
wc.status);
break;
default:
- printk(KERN_ERR "Unknown completion: %d
with error\n", wc.opcode);
+ printk(KERN_ERR "Unknown completion %d
with error %d\n", wc.opcode, wc.status);
break;
}
err_status = 1;
@@ -844,9 +861,9 @@
ib_mad_recv_done_handler(port_priv, &wc);
break;
default:
- printk(KERN_ERR "Wrong Opcode: %d\n",
wc.opcode);
+ printk(KERN_ERR "Wrong Opcode %d on
completion\n", wc.opcode);
if (wc.status) {
- printk(KERN_ERR "Completion error:
%d\n", wc.status);
+ printk(KERN_ERR "Completion error %d\n",
wc.status);
}
}
@@ -855,7 +872,6 @@
if (err_status) {
ib_mad_port_restart(port_priv);
} else {
- ib_mad_post_receive_mads(port_priv);
ib_req_notify_cq(port_priv->cq, IB_CQ_NEXT_COMP);
}
}
@@ -871,7 +887,9 @@
while (1) {
while (!signal_pending(current)) {
- ret =
wait_event_interruptible(mad_thread_priv->wait, 0);
+ ret =
wait_event_interruptible(mad_thread_priv->wait,
+
atomic_read(&mad_thread_priv->completion_event) > 0);
+ atomic_set(&mad_thread_priv->completion_event,
0);
if (ret) {
printk(KERN_ERR "ib_mad thread
exiting\n");
return 0;
@@ -890,6 +908,7 @@
{
struct ib_mad_thread_private *mad_thread_priv =
&port_priv->mad_thread_private;
+ atomic_set(&mad_thread_priv->completion_event, 0);
init_waitqueue_head(&mad_thread_priv->wait);
port_priv->mad_thread = kthread_create(ib_mad_thread,
@@ -898,10 +917,11 @@
port_priv->device->name,
port_priv->port_num);
if (IS_ERR(port_priv->mad_thread)) {
- printk(KERN_ERR "Couldn't start mad thread for %s port
%d\n",
+ printk(KERN_ERR "Couldn't start ib_mad thread for %s
port %d\n",
port_priv->device->name, port_priv->port_num);
return 1;
}
+ wake_up_process(port_priv->mad_thread);
return 0;
}
@@ -918,6 +938,7 @@
struct ib_mad_port_private *port_priv = cq->cq_context;
struct ib_mad_thread_private *mad_thread_priv =
&port_priv->mad_thread_private;
+ atomic_inc(&mad_thread_priv->completion_event);
wake_up_interruptible(&mad_thread_priv->wait);
}
@@ -930,7 +951,16 @@
struct ib_recv_wr *bad_recv_wr;
unsigned long flags;
int ret;
+ union ib_mad_recv_wrid wrid;
+ int qpn;
+
+ qpn = convert_qpnum(qp->qp_num);
+ if (qpn == -1) {
+ printk(KERN_ERR "Post receive to invalid QPN %d\n",
qp->qp_num);
+ return -EINVAL;
+ }
+
/*
* Allocate memory for receive buffer.
* This is for both MAD and private header
@@ -949,7 +979,7 @@
/* Setup scatter list */
sg_list.addr = pci_map_single(port_priv->device->dma_device,
&mad_priv->grh,
- sizeof *mad_priv - sizeof
mad_priv->header,
+ sizeof *mad_priv - sizeof
mad_priv->header,
PCI_DMA_FROMDEVICE);
sg_list.length = sizeof *mad_priv - sizeof mad_priv->header;
sg_list.lkey = (*port_priv->mr).lkey;
@@ -959,13 +989,15 @@
recv_wr.sg_list = &sg_list;
recv_wr.num_sge = 1;
recv_wr.recv_flags = IB_RECV_SIGNALED;
- recv_wr.wr_id = qp->qp_num; /* 32 bits left */
+ wrid.wrid_field.index = port_priv->recv_wr_index[qpn]++;
+ wrid.wrid_field.qpn = qp->qp_num;
+ recv_wr.wr_id = wrid.wrid;
/* Link receive WR into posted receive MAD list */
spin_lock_irqsave(&port_priv->recv_list_lock, flags);
list_add_tail(&mad_priv->header.recv_buf.list,
-
&port_priv->recv_posted_mad_list[convert_qpnum(qp->qp_num)]);
- port_priv->recv_posted_mad_count[convert_qpnum(qp->qp_num)]++;
+ &port_priv->recv_posted_mad_list[qpn]);
+ port_priv->recv_posted_mad_count[qpn]++;
spin_unlock_irqrestore(&port_priv->recv_list_lock, flags);
pci_unmap_addr_set(&mad_priv->header, mapping, sg_list.addr);
@@ -982,11 +1014,11 @@
/* Unlink from posted receive MAD list */
spin_lock_irqsave(&port_priv->recv_list_lock, flags);
list_del(&mad_priv->header.recv_buf.list);
-
port_priv->recv_posted_mad_count[convert_qpnum(qp->qp_num)]--;
+ port_priv->recv_posted_mad_count[qpn]--;
spin_unlock_irqrestore(&port_priv->recv_list_lock,
flags);
kmem_cache_free(ib_mad_cache, mad_priv);
- printk(KERN_NOTICE "ib_post_recv failed ret = %d\n",
ret);
+ printk(KERN_NOTICE "ib_post_recv WRID 0x%Lx failed ret =
%d\n", recv_wr.wr_id, ret);
return -EINVAL;
}
@@ -1028,6 +1060,8 @@
/* PCI mapping !!! */
+ list_del(&port_priv->recv_posted_mad_list[i]);
+
}
INIT_LIST_HEAD(&port_priv->recv_posted_mad_list[i]);
port_priv->recv_posted_mad_count[i] = 0;
@@ -1045,11 +1079,9 @@
spin_lock_irqsave(&port_priv->send_list_lock, flags);
while (!list_empty(&port_priv->send_posted_mad_list)) {
- /* PCI mapping ? */
-
list_del(&port_priv->send_posted_mad_list);
- /* Call completion handler with some status ? */
+ /* Call completion handler with flushed status !!! */
}
INIT_LIST_HEAD(&port_priv->send_posted_mad_list);
@@ -1345,6 +1377,7 @@
qp_init_attr.cap.max_recv_wr = IB_MAD_QP_RECV_SIZE;
qp_init_attr.cap.max_send_sge = IB_MAD_SEND_REQ_MAX_SG;
qp_init_attr.cap.max_recv_sge = IB_MAD_RECV_REQ_MAX_SG;
+ /* Until Roland's ib_verbs.h ip_qp_types enum reordered
!!! */
if (i == 0)
qp_init_attr.qp_type = IB_QPT_SMI;
else
@@ -1373,6 +1406,7 @@
for (i = 0; i < IB_MAD_QPS_SUPPORTED; i++) {
INIT_LIST_HEAD(&port_priv->recv_posted_mad_list[i]);
port_priv->recv_posted_mad_count[i] = 0;
+ port_priv->recv_wr_index[i] = 0;
}
ret = ib_mad_thread_init(port_priv);
More information about the general
mailing list