[ofa-general] [PATCH] sdp: Interrupts performance fixes

[Amir Vadai]

* use rcvbuf_initial - fix server to use rcvbuf_initial_size as initial
* hw interrupt moderation
* fix interrupts statistics
* fix max frags bug

Signed-off-by: Amir Vadai <amirv at mellanox.co.il>
---
 drivers/infiniband/ulp/sdp/sdp.h       |    7 +++++--
 drivers/infiniband/ulp/sdp/sdp_bcopy.c |    1 +
 drivers/infiniband/ulp/sdp/sdp_cma.c   |   21 +++++++++++++++------
 drivers/infiniband/ulp/sdp/sdp_proc.c  |    3 +--
 4 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/drivers/infiniband/ulp/sdp/sdp.h b/drivers/infiniband/ulp/sdp/sdp.h
index e36f786..a4d3aa2 100644
--- a/drivers/infiniband/ulp/sdp/sdp.h
+++ b/drivers/infiniband/ulp/sdp/sdp.h
@@ -83,8 +83,7 @@ struct sdpstats {
 	u32 sendmsg_seglen[25];
 	u32 send_size[25];
 	u32 post_recv;
-	u32 rx_int_count;
-	u32 tx_int_count;
+	u32 int_count;
 	u32 bzcopy_poll_miss;
 	u32 send_wait_for_mem;
 	u32 send_miss_no_credits;
@@ -152,6 +151,10 @@ static inline void sdpstats_hist(u32 *h, u32 val, u32 maxidx, int is_log)
 #define SDP_OP_RECV 0x800000000LL
 #define SDP_OP_SEND 0x400000000LL
 
+#ifndef MIN
+#define MIN(a, b) (a < b ? a : b)
+#endif
+
 extern struct list_head sock_list;
 extern spinlock_t sock_list_lock;
 
diff --git a/drivers/infiniband/ulp/sdp/sdp_bcopy.c b/drivers/infiniband/ulp/sdp/sdp_bcopy.c
index 7c7271c..c5a0ccf 100644
--- a/drivers/infiniband/ulp/sdp/sdp_bcopy.c
+++ b/drivers/infiniband/ulp/sdp/sdp_bcopy.c
@@ -917,6 +917,7 @@ void sdp_completion_handler(struct ib_cq *cq, void *cq_context)
 	struct sock *sk = cq_context;
 	struct sdp_sock *ssk = sdp_sk(sk);
 	schedule_work(&ssk->work);
+	SDPSTATS_COUNTER_INC(int_count);
 }
 
 int sdp_poll_cq(struct sdp_sock *ssk, struct ib_cq *cq)
diff --git a/drivers/infiniband/ulp/sdp/sdp_cma.c b/drivers/infiniband/ulp/sdp/sdp_cma.c
index b6bc4e5..3bc2b02 100644
--- a/drivers/infiniband/ulp/sdp/sdp_cma.c
+++ b/drivers/infiniband/ulp/sdp/sdp_cma.c
@@ -129,7 +129,12 @@ static int sdp_init_qp(struct sock *sk, struct rdma_cm_id *id)
 		goto err_cq;
 	}
 
-	ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
+	rc = ib_modify_cq(cq, 10, 200);
+	if (rc) {
+		sdp_warn(sk, "Unable to modify RX CQ: %d.\n", rc);
+		goto err_qp;
+	}
+	sdp_warn(sk, "Initialized CQ moderation\n");
 
         qp_init_attr.send_cq = qp_init_attr.recv_cq = cq;
 
@@ -207,10 +212,11 @@ static int sdp_connect_handler(struct sock *sk, struct rdma_cm_id *id,
 		sizeof(struct sdp_bsdh);
 	sdp_sk(child)->send_frags = PAGE_ALIGN(sdp_sk(child)->xmit_size_goal) /
 		PAGE_SIZE;
-	sdp_init_buffers(sdp_sk(child), ntohl(h->desremrcvsz));
+        sdp_init_buffers(sdp_sk(child), rcvbuf_initial_size);
 
-	sdp_dbg(child, "%s bufs %d xmit_size_goal %d send trigger %d\n",
+	sdp_dbg(child, "%s recv_frags: %d bufs %d xmit_size_goal %d send trigger %d\n",
 		__func__,
+		sdp_sk(child)->recv_frags,
 		sdp_sk(child)->bufs,
 		sdp_sk(child)->xmit_size_goal,
 		sdp_sk(child)->min_bufs);
@@ -251,13 +257,16 @@ static int sdp_response_handler(struct sock *sk, struct rdma_cm_id *id,
 	sdp_sk(sk)->min_bufs = sdp_sk(sk)->bufs / 4;
 	sdp_sk(sk)->xmit_size_goal = ntohl(h->actrcvsz) -
 		sizeof(struct sdp_bsdh);
-	sdp_sk(sk)->send_frags = PAGE_ALIGN(sdp_sk(sk)->xmit_size_goal) /
-		PAGE_SIZE;
+ 	sdp_sk(sk)->send_frags = MIN(PAGE_ALIGN(sdp_sk(sk)->xmit_size_goal) /
+ 		PAGE_SIZE, SDP_MAX_SEND_SKB_FRAGS);
+ 	sdp_sk(sk)->xmit_size_goal = MIN(sdp_sk(sk)->xmit_size_goal, 
+ 		sdp_sk(sk)->send_frags * PAGE_SIZE);
 
-	sdp_dbg(sk, "%s bufs %d xmit_size_goal %d send trigger %d\n",
+	sdp_dbg(sk, "%s bufs %d xmit_size_goal %d send_frags: %d send trigger %d\n",
 		__func__,
 		sdp_sk(sk)->bufs,
 		sdp_sk(sk)->xmit_size_goal,
+ 		sdp_sk(sk)->send_frags,
 		sdp_sk(sk)->min_bufs);
 
 	sdp_sk(sk)->poll_cq = 1;
diff --git a/drivers/infiniband/ulp/sdp/sdp_proc.c b/drivers/infiniband/ulp/sdp/sdp_proc.c
index 8971517..b8614a5 100644
--- a/drivers/infiniband/ulp/sdp/sdp_proc.c
+++ b/drivers/infiniband/ulp/sdp/sdp_proc.c
@@ -262,8 +262,7 @@ static int sdpstats_seq_show(struct seq_file *seq, void *v)
 	seq_printf(seq, "tx_poll_miss      \t\t: %d\n", sdpstats.tx_poll_miss);
 
 	seq_printf(seq, "CQ stats:\n");
-	seq_printf(seq, "- RX interrupts\t\t: %d\n", sdpstats.rx_int_count);
-	seq_printf(seq, "- TX interrupts\t\t: %d\n", sdpstats.tx_int_count);
+	seq_printf(seq, "- interrupts\t\t: %d\n", sdpstats.int_count);
 	return 0;
 }
 
-- 
1.5.3.7