[RESENT][ofa-general] [PATCH 2/3] ib/ipoib: set IPoIB-UD RX S/G parameters

[Shirley Ma]

Signed-off-by: Shirley Ma <xma at us.ibm.com>

diff --git a/drivers/infiniband/ulp/ipoib/ipoib.h
b/drivers/infiniband/ulp/ipoib/ipoib.h
index 138f1a3..65b1159 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib.h
+++ b/drivers/infiniband/ulp/ipoib/ipoib.h
@@ -56,11 +56,11 @@
/* constants */

enum {
- IPOIB_PACKET_SIZE   = 2048,
- IPOIB_BUF_SIZE   = IPOIB_PACKET_SIZE + IB_GRH_BYTES,
-
IPOIB_ENCAP_LEN   = 4,

+ IPOIB_MAX_IB_MTU   = 4096, /* max ib device payload is 4096 */
+ IPOIB_UD_MAX_RX_SG   = ALIGN(IPOIB_MAX_IB_MTU + IB_GRH_BYTES + 4,
PAGE_SIZE) / PAGE_SIZE,  /* padding to align IP header */
+
IPOIB_CM_MTU   = 0x10000 - 0x10, /* padding to align header to 16 */
IPOIB_CM_BUF_SIZE   = IPOIB_CM_MTU  + IPOIB_ENCAP_LEN,
IPOIB_CM_HEAD_SIZE   = IPOIB_CM_BUF_SIZE % PAGE_SIZE,
@@ -314,6 +314,9 @@ struct ipoib_dev_priv {
struct dentry *mcg_dentry;
struct dentry *path_dentry;
#endif
+ int max_ib_mtu;
+ struct ib_sge rx_sge[IPOIB_UD_MAX_RX_SG];
+ struct ib_recv_wr rx_wr;
};

struct ipoib_ah {
@@ -354,6 +357,11 @@ struct ipoib_neigh {
struct list_head    list;
};

+#define IPOIB_UD_MTU(ib_mtu) (ib_mtu - IPOIB_ENCAP_LEN)
+#define IPOIB_UD_BUF_SIZE(ib_mtu) (ib_mtu + IB_GRH_BYTES + 4) /*
padding to align IP header */
+#define IPOIB_UD_HEAD_SIZE(ib_mtu) (IPOIB_UD_BUF_SIZE(ib_mtu)) %
PAGE_SIZE
+#define IPOIB_UD_RX_SG(ib_mtu) ALIGN(IPOIB_UD_BUF_SIZE(ib_mtu),
PAGE_SIZE) / PAGE_SIZE
+
/*
  * We stash a pointer to our private neighbour information after our
  * hardware address in neigh->ha.  The ALIGN() expression here makes
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c
b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index a082466..646aeb2 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -194,7 +194,7 @@ static int ipoib_change_mtu(struct net_device *dev,
int new_mtu)
return 0;
}

- if (new_mtu > IPOIB_PACKET_SIZE - IPOIB_ENCAP_LEN)
+ if (new_mtu > IPOIB_UD_MTU(priv->max_ib_mtu))
return -EINVAL;

priv->admin_mtu = new_mtu;
@@ -968,10 +968,6 @@ static void ipoib_setup(struct net_device *dev)
dev->tx_queue_len = ipoib_sendq_size * 2;
dev->features = NETIF_F_VLAN_CHALLENGED | NETIF_F_LLTX;

- /* MTU will be reset when mcast join happens */
- dev->mtu = IPOIB_PACKET_SIZE - IPOIB_ENCAP_LEN;
- priv->mcast_mtu = priv->admin_mtu = dev->mtu;
-
memcpy(dev->broadcast, ipv4_bcast_addr, INFINIBAND_ALEN);

netif_carrier_off(dev);
@@ -1103,6 +1099,7 @@ static struct net_device *ipoib_add_port(const
char *format,
struct ib_device *hca, u8 port)
{
struct ipoib_dev_priv *priv;
+ struct ib_port_attr attr;
int result = -ENOMEM;

priv = ipoib_intf_alloc(format);
@@ -1111,6 +1108,18 @@ static struct net_device *ipoib_add_port(const
char *format,

SET_NETDEV_DEV(priv->dev, hca->dma_device);

+ if (!ib_query_port(hca, port, &attr))
+ priv->max_ib_mtu = ib_mtu_enum_to_int(attr.max_mtu);
+ else {
+ printk(KERN_WARNING "%s: ib_query_port %d failed\n",
+        hca->name, port);
+ goto device_init_failed;
+ } 
+ 
+ /* MTU will be reset when mcast join happens */
+ priv->dev->mtu  = IPOIB_UD_MTU(priv->max_ib_mtu);
+ priv->mcast_mtu  = priv->admin_mtu = priv->dev->mtu;
+
result = ib_query_pkey(hca, port, 0, &priv->pkey);
if (result) {
printk(KERN_WARNING "%s: ib_query_pkey port %d failed (ret = %d)\n",
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
index 2628339..630b429 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
@@ -567,8 +567,7 @@ void ipoib_mcast_join_task(struct work_struct *work)
return;
}

- priv->mcast_mtu = ib_mtu_enum_to_int(priv->broadcast->mcmember.mtu) -
- IPOIB_ENCAP_LEN;
+ priv->mcast_mtu =
IPOIB_UD_MTU(ib_mtu_enum_to_int(priv->broadcast->mcmember.mtu));

if (!ipoib_cm_admin_enabled(dev))
dev->mtu = min(priv->mcast_mtu, priv->admin_mtu);
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
index 433e99a..eefdb6a 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
@@ -150,7 +150,7 @@ int ipoib_transport_dev_init(struct net_device *dev,
struct ib_device *ca)
.max_send_wr  = ipoib_sendq_size,
.max_recv_wr  = ipoib_recvq_size,
.max_send_sge = 1,
- .max_recv_sge = 1
+ .max_recv_sge = IPOIB_UD_RX_SG(priv->max_ib_mtu) 
},
.sq_sig_type = IB_SIGNAL_ALL_WR,
.qp_type     = IB_QPT_UD
@@ -208,6 +208,16 @@ int ipoib_transport_dev_init(struct net_device
*dev, struct ib_device *ca)
priv->tx_wr.num_sge = 1;
priv->tx_wr.send_flags = IB_SEND_SIGNALED;

+ priv->rx_sge[0].length = IPOIB_UD_HEAD_SIZE(priv->max_ib_mtu);
+ for (i = 0; i < IPOIB_UD_RX_SG(priv->max_ib_mtu) - 1; ++i) {
+ priv->rx_sge[i].lkey = priv->mr->lkey;
+ priv->rx_sge[i + 1].length = PAGE_SIZE;
+ }
+ priv->rx_sge[i + 1].lkey = priv->mr->lkey;
+ priv->rx_wr.num_sge = IPOIB_UD_RX_SG(priv->max_ib_mtu);
+ priv->rx_wr.next = NULL;
+ priv->rx_wr.sg_list = priv->rx_sge;
+
return 0;

out_free_cq: