[ofa-general] [PATCH 1/2] RDMA/nes: Register memory for LSMM

[Faisal Latif]

The LSMM is using STAG0. Modify the driver to register LSMM and
use the non-zero STAG in nes_accept(). Deregister it from nes_disconnect() as
it is called during connection termination.

Signed-off-by: Faisal Latif <faisal.latif at intel.com>
---
 drivers/infiniband/hw/nes/nes_cm.c    |   39 ++++++++++++++++++++++++++++----
 drivers/infiniband/hw/nes/nes_verbs.c |    2 +
 drivers/infiniband/hw/nes/nes_verbs.h |    1 +
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c
index bd918df..6681d57 100644
--- a/drivers/infiniband/hw/nes/nes_cm.c
+++ b/drivers/infiniband/hw/nes/nes_cm.c
@@ -2486,12 +2486,14 @@ static int nes_disconnect(struct nes_qp *nesqp, int abrupt)
 	int ret = 0;
 	struct nes_vnic *nesvnic;
 	struct nes_device *nesdev;
+	struct nes_ib_device *nesibdev;
 
 	nesvnic = to_nesvnic(nesqp->ibqp.device);
 	if (!nesvnic)
 		return -EINVAL;
 
 	nesdev = nesvnic->nesdev;
+	nesibdev = nesvnic->nesibdev;
 
 	nes_debug(NES_DBG_CM, "netdev refcnt = %u.\n",
 			atomic_read(&nesvnic->netdev->refcnt));
@@ -2503,6 +2505,8 @@ static int nes_disconnect(struct nes_qp *nesqp, int abrupt)
 	} else {
 		/* Need to free the Last Streaming Mode Message */
 		if (nesqp->ietf_frame) {
+			if (nesqp->lsmm_mr)
+				nesibdev->ibdev.dereg_mr(nesqp->lsmm_mr);
 			pci_free_consistent(nesdev->pcidev,
 					nesqp->private_data_len+sizeof(struct ietf_mpa_frame),
 					nesqp->ietf_frame, nesqp->ietf_frame_pbase);
@@ -2539,6 +2543,12 @@ int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 	u32 crc_value;
 	int ret;
 	int passive_state;
+	struct nes_ib_device *nesibdev;
+	struct ib_mr *ibmr = NULL;
+	struct ib_phys_buf ibphysbuf;
+	struct nes_pd *nespd;
+
+
 
 	ibqp = nes_get_qp(cm_id->device, conn_param->qpn);
 	if (!ibqp)
@@ -2597,6 +2607,26 @@ int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 	if (cm_id->remote_addr.sin_addr.s_addr !=
 			cm_id->local_addr.sin_addr.s_addr) {
 		u64temp = (unsigned long)nesqp;
+		nesibdev = nesvnic->nesibdev;
+		nespd = nesqp->nespd;
+		ibphysbuf.addr = nesqp->ietf_frame_pbase;
+		ibphysbuf.size = conn_param->private_data_len +
+					sizeof(struct ietf_mpa_frame);
+		ibmr = nesibdev->ibdev.reg_phys_mr((struct ib_pd *)nespd,
+						&ibphysbuf, 1,
+						IB_ACCESS_LOCAL_WRITE,
+						(u64 *)&nesqp->ietf_frame);
+		if (!ibmr) {
+			nes_debug(NES_DBG_CM, "Unable to register memory region"
+					"for lSMM for cm_node = %p \n",
+					cm_node);
+			return -ENOMEM;
+		}
+
+		ibmr->pd = &nespd->ibpd;
+		ibmr->device = nespd->ibpd.device;
+		nesqp->lsmm_mr = ibmr;
+
 		u64temp |= NES_SW_CONTEXT_ALIGN>>1;
 		set_wqe_64bit_value(wqe->wqe_words,
 			NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX,
@@ -2607,14 +2637,13 @@ int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 		wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] =
 			cpu_to_le32(conn_param->private_data_len +
 			sizeof(struct ietf_mpa_frame));
-		wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_LOW_IDX] =
-			cpu_to_le32((u32)nesqp->ietf_frame_pbase);
-		wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX] =
-			cpu_to_le32((u32)((u64)nesqp->ietf_frame_pbase >> 32));
+		set_wqe_64bit_value(wqe->wqe_words,
+					NES_IWARP_SQ_WQE_FRAG0_LOW_IDX,
+					(u64)nesqp->ietf_frame);
 		wqe->wqe_words[NES_IWARP_SQ_WQE_LENGTH0_IDX] =
 			cpu_to_le32(conn_param->private_data_len +
 			sizeof(struct ietf_mpa_frame));
-		wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX] = 0;
+		wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX] = ibmr->lkey;
 
 		nesqp->nesqp_context->ird_ord_sizes |=
 			cpu_to_le32(NES_QPCONTEXT_ORDIRD_LSMM_PRESENT |
diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c
index b42b17a..6e65c86 100644
--- a/drivers/infiniband/hw/nes/nes_verbs.c
+++ b/drivers/infiniband/hw/nes/nes_verbs.c
@@ -1363,8 +1363,10 @@ static struct ib_qp *nes_create_qp(struct ib_pd *ibpd,
 					NES_QPCONTEXT_MISC_RQ_SIZE_SHIFT);
 			nesqp->nesqp_context->misc |= cpu_to_le32((u32)nesqp->hwqp.sq_encoded_size <<
 					NES_QPCONTEXT_MISC_SQ_SIZE_SHIFT);
+			if (!udata) {
 				nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_PRIV_EN);
 				nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_FAST_REGISTER_EN);
+			}
 			nesqp->nesqp_context->cqs = cpu_to_le32(nesqp->nesscq->hw_cq.cq_number +
 					((u32)nesqp->nesrcq->hw_cq.cq_number << 16));
 			u64temp = (u64)nesqp->hwqp.sq_pbase;
diff --git a/drivers/infiniband/hw/nes/nes_verbs.h b/drivers/infiniband/hw/nes/nes_verbs.h
index da3c368..5e48f67 100644
--- a/drivers/infiniband/hw/nes/nes_verbs.h
+++ b/drivers/infiniband/hw/nes/nes_verbs.h
@@ -134,6 +134,7 @@ struct nes_qp {
 	struct ietf_mpa_frame *ietf_frame;
 	dma_addr_t            ietf_frame_pbase;
 	wait_queue_head_t     state_waitq;
+	struct ib_mr          *lsmm_mr;
 	unsigned long         socket;
 	struct nes_hw_qp      hwqp;
 	struct work_struct    work;
-- 
1.5.3.3