[openib-general] [PATCH 2 of 3] mad: large RMPP support
Jack Morgenstein
jackm at mellanox.co.il
Mon Feb 6 23:40:36 PST 2006
patch 2 of 3
---
Large RMPP support, receive side: copy the arriving MADs to chunks
instead of coalescing to one large buffer in kernel space.
Signed-off-by: Jack Morgenstein <jackm at mellanox.co.il>
Signed-off-by: Michael S. Tsirkin <mst at mellanox.co.il>
Index: last_stable/drivers/infiniband/core/mad_rmpp.c
===================================================================
--- last_stable.orig/drivers/infiniband/core/mad_rmpp.c
+++ last_stable/drivers/infiniband/core/mad_rmpp.c
@@ -433,44 +433,6 @@ static struct ib_mad_recv_wc * complete_
return rmpp_wc;
}
-void ib_coalesce_recv_mad(struct ib_mad_recv_wc *mad_recv_wc, void *buf)
-{
- struct ib_mad_recv_buf *seg_buf;
- struct ib_rmpp_mad *rmpp_mad;
- void *data;
- int size, len, offset;
- u8 flags;
-
- len = mad_recv_wc->mad_len;
- if (len <= sizeof(struct ib_mad)) {
- memcpy(buf, mad_recv_wc->recv_buf.mad, len);
- return;
- }
-
- offset = data_offset(mad_recv_wc->recv_buf.mad->mad_hdr.mgmt_class);
-
- list_for_each_entry(seg_buf, &mad_recv_wc->rmpp_list, list) {
- rmpp_mad = (struct ib_rmpp_mad *)seg_buf->mad;
- flags = ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr);
-
- if (flags & IB_MGMT_RMPP_FLAG_FIRST) {
- data = rmpp_mad;
- size = sizeof(*rmpp_mad);
- } else {
- data = (void *) rmpp_mad + offset;
- if (flags & IB_MGMT_RMPP_FLAG_LAST)
- size = len;
- else
- size = sizeof(*rmpp_mad) - offset;
- }
-
- memcpy(buf, data, size);
- len -= size;
- buf += size;
- }
-}
-EXPORT_SYMBOL(ib_coalesce_recv_mad);
-
static struct ib_mad_recv_wc *
continue_rmpp(struct ib_mad_agent_private *agent,
struct ib_mad_recv_wc *mad_recv_wc)
Index: last_stable/drivers/infiniband/core/user_mad.c
===================================================================
--- last_stable.orig/drivers/infiniband/core/user_mad.c
+++ last_stable/drivers/infiniband/core/user_mad.c
@@ -176,6 +177,88 @@ static int queue_packet(struct ib_umad_f
return ret;
}
+static int data_offset(u8 mgmt_class)
+{
+ if (mgmt_class == IB_MGMT_CLASS_SUBN_ADM)
+ return IB_MGMT_SA_HDR;
+ else if ((mgmt_class >= IB_MGMT_CLASS_VENDOR_RANGE2_START) &&
+ (mgmt_class <= IB_MGMT_CLASS_VENDOR_RANGE2_END))
+ return IB_MGMT_VENDOR_HDR;
+ else
+ return IB_MGMT_RMPP_HDR;
+}
+
+static int copy_recv_mad(struct ib_mad_recv_wc *mad_recv_wc,
+ struct ib_umad_packet *packet)
+{
+ struct ib_mad_recv_buf *seg_buf;
+ struct ib_rmpp_mad *rmpp_mad;
+ void *data;
+ struct ib_mad_multipacket_seg *seg;
+ int size, len, offset;
+ u8 flags;
+
+ len = mad_recv_wc->mad_len;
+ if (len <= sizeof(struct ib_mad)) {
+ memcpy(&packet->mad.data, mad_recv_wc->recv_buf.mad, len);
+ return 0;
+ }
+
+ /* Multipacket (RMPP) MAD */
+ offset = data_offset(mad_recv_wc->recv_buf.mad->mad_hdr.mgmt_class);
+
+ list_for_each_entry(seg_buf, &mad_recv_wc->rmpp_list, list) {
+ rmpp_mad = (struct ib_rmpp_mad *)seg_buf->mad;
+ flags = ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr);
+
+ if (flags & IB_MGMT_RMPP_FLAG_FIRST) {
+ size = sizeof(*rmpp_mad);
+ memcpy(&packet->mad.data, rmpp_mad, size);
+ } else {
+ data = (void *) rmpp_mad + offset;
+ if (flags & IB_MGMT_RMPP_FLAG_LAST)
+ size = len;
+ else
+ size = sizeof(*rmpp_mad) - offset;
+ seg = kmalloc(sizeof(struct ib_mad_multipacket_seg) +
+ sizeof(struct ib_rmpp_mad) - offset,
+ GFP_KERNEL);
+ if (!seg)
+ return -ENOMEM;
+ memcpy(seg->data, data, size);
+ list_add_tail(&seg->list, &packet->seg_list);
+ }
+ len -= size;
+ }
+ return 0;
+}
+
+static struct ib_umad_packet *alloc_packet(void)
+{
+ struct ib_umad_packet *packet;
+ int length = sizeof *packet + sizeof(struct ib_mad);
+
+ packet = kzalloc(length, GFP_KERNEL);
+ if (!packet) {
+ printk(KERN_ERR "alloc_packet: mem alloc failed for length %d\n",
+ length);
+ return NULL;
+ }
+ INIT_LIST_HEAD(&packet->seg_list);
+ return packet;
+}
+
+static void free_packet(struct ib_umad_packet *packet)
+{
+ struct ib_mad_multipacket_seg *seg, *tmp;
+
+ list_for_each_entry_safe(seg, tmp, &packet->seg_list, list) {
+ list_del(&seg->list);
+ kfree(seg);
+ }
+ kfree(packet);
+}
+
static void send_handler(struct ib_mad_agent *agent,
struct ib_mad_send_wc *send_wc)
{
@@ -243,13 +298,16 @@ static void recv_handler(struct ib_mad_a
goto out;
length = mad_recv_wc->mad_len;
- packet = alloc_packet(length);
+ packet = alloc_packet();
if (!packet)
goto out;
packet->length = length;
- ib_coalesce_recv_mad(mad_recv_wc, packet->mad.data);
+ if (copy_recv_mad(mad_recv_wc, packet)) {
+ free_packet(packet);
+ goto out;
+ }
packet->mad.hdr.status = 0;
packet->mad.hdr.length = length + sizeof (struct ib_user_mad);
@@ -278,6 +336,7 @@ static ssize_t ib_umad_read(struct file
size_t count, loff_t *pos)
{
struct ib_umad_file *file = filp->private_data;
+ struct ib_mad_multipacket_seg *seg;
struct ib_umad_packet *packet;
ssize_t ret;
@@ -304,18 +363,42 @@ static ssize_t ib_umad_read(struct file
spin_unlock_irq(&file->recv_lock);
- if (count < packet->length + sizeof (struct ib_user_mad)) {
- /* Return length needed (and first RMPP segment) if too small */
- if (copy_to_user(buf, &packet->mad,
- sizeof (struct ib_user_mad) + sizeof (struct ib_mad)))
- ret = -EFAULT;
- else
- ret = -ENOSPC;
- } else if (copy_to_user(buf, &packet->mad,
- packet->length + sizeof (struct ib_user_mad)))
+ if (copy_to_user(buf, &packet->mad,
+ sizeof(struct ib_user_mad) + sizeof(struct ib_mad))) {
ret = -EFAULT;
- else
+ goto err;
+ }
+
+ if (count < packet->length + sizeof (struct ib_user_mad))
+ /* User buffer too small. Return first RMPP segment (which
+ * includes RMPP message length).
+ */
+ ret = -ENOSPC;
+ else if (packet->length <= sizeof(struct ib_mad))
+ ret = packet->length + sizeof(struct ib_user_mad);
+ else {
+ int len = packet->length - sizeof(struct ib_mad);
+ struct ib_rmpp_mad *rmpp_mad =
+ (struct ib_rmpp_mad *) packet->mad.data;
+ int max_seg_payload = sizeof(struct ib_mad) -
+ data_offset(rmpp_mad->mad_hdr.mgmt_class);
+ int seg_payload;
+ /* multipacket RMPP MAD message. Copy remainder of message.
+ * Note that last segment may have a shorter payload.
+ */
+ buf += sizeof(struct ib_user_mad) + sizeof(struct ib_mad);
+ list_for_each_entry(seg, &packet->seg_list, list) {
+ seg_payload = min_t(int, len, max_seg_payload);
+ if (copy_to_user(buf, seg->data, seg_payload)) {
+ ret = -EFAULT;
+ goto err;
+ }
+ buf += seg_payload;
+ len -= seg_payload;
+ }
ret = packet->length + sizeof (struct ib_user_mad);
+ }
+err:
if (ret < 0) {
/* Requeue packet */
spin_lock_irq(&file->recv_lock);
More information about the general
mailing list