[openib-general] [PATCH] mthca: query_qp and query_srq
Eli Cohen
eli at mellanox.co.il
Sun Feb 12 23:27:37 PST 2006
Signed-off-by: Eli Cohen <eli at mellanox.co.il>
Signed-off-by: Michael S. Tsirkin <mst at mellanox.co.il>
implement ib_query_qp() and ib_query_srq() for mthca.
Index: openib_gen2/drivers/infiniband/hw/mthca/mthca_cmd.h
===================================================================
--- openib_gen2.orig/drivers/infiniband/hw/mthca/mthca_cmd.h
+++ openib_gen2/drivers/infiniband/hw/mthca/mthca_cmd.h
@@ -303,6 +303,8 @@ int mthca_RESIZE_CQ(struct mthca_dev *de
u8 *status);
int mthca_SW2HW_SRQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox,
int srq_num, u8 *status);
+int mthca_QUERY_SRQ(struct mthca_dev *dev, u32 num,
+ struct mthca_mailbox *mailbox, u8 *status);
int mthca_HW2SW_SRQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox,
int srq_num, u8 *status);
int mthca_ARM_SRQ(struct mthca_dev *dev, int srq_num, int limit, u8 *status);
Index: openib_gen2/drivers/infiniband/core/verbs.c
===================================================================
--- openib_gen2.orig/drivers/infiniband/core/verbs.c
+++ openib_gen2/drivers/infiniband/core/verbs.c
@@ -257,9 +257,18 @@ int ib_query_qp(struct ib_qp *qp,
int qp_attr_mask,
struct ib_qp_init_attr *qp_init_attr)
{
- return qp->device->query_qp ?
+ int err;
+
+ err = qp->device->query_qp ?
qp->device->query_qp(qp, qp_attr, qp_attr_mask, qp_init_attr) :
-ENOSYS;
+ if (err)
+ return err;
+ qp_init_attr->recv_cq = qp->recv_cq;
+ qp_init_attr->send_cq = qp->send_cq;
+ qp_init_attr->srq = qp->srq;
+
+ return err;
}
EXPORT_SYMBOL(ib_query_qp);
Index: openib_gen2/drivers/infiniband/hw/mthca/mthca_srq.c
===================================================================
--- openib_gen2.orig/drivers/infiniband/hw/mthca/mthca_srq.c
+++ openib_gen2/drivers/infiniband/hw/mthca/mthca_srq.c
@@ -357,6 +357,40 @@ int mthca_modify_srq(struct ib_srq *ibsr
return 0;
}
+
+int mthca_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
+{
+ struct mthca_dev *dev = to_mdev(ibsrq->device);
+ struct mthca_srq *srq = to_msrq(ibsrq);
+ struct mthca_mailbox *mailbox;
+ struct mthca_arbel_srq_context *arbel_ctx;
+ u8 status;
+ int err;
+
+ mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
+ if (IS_ERR(mailbox))
+ return PTR_ERR(mailbox);
+
+ err = mthca_QUERY_SRQ(dev, srq->srqn, mailbox, &status);
+ if (err)
+ goto exit;
+ if (mthca_is_memfree(dev)) {
+ arbel_ctx = mailbox->buf;
+ srq_attr->srq_limit = arbel_ctx->limit_watermark;
+ }
+ else
+ srq_attr->srq_limit = 0;
+
+ srq_attr->pd = ibsrq->pd;
+ srq_attr->max_wr = srq->max;
+ srq_attr->max_sge = srq->max_gs;
+
+exit:
+ mthca_free_mailbox(dev, mailbox);
+
+ return err;
+}
+
void mthca_srq_event(struct mthca_dev *dev, u32 srqn,
enum ib_event_type event_type)
{
Index: openib_gen2/drivers/infiniband/include/rdma/ib_verbs.h
===================================================================
--- openib_gen2.orig/drivers/infiniband/include/rdma/ib_verbs.h
+++ openib_gen2/drivers/infiniband/include/rdma/ib_verbs.h
@@ -423,9 +423,10 @@ enum ib_srq_attr_mask {
};
struct ib_srq_attr {
- u32 max_wr;
- u32 max_sge;
- u32 srq_limit;
+ u32 max_wr;
+ u32 max_sge;
+ u32 srq_limit;
+ struct ib_pd *pd;
};
struct ib_srq_init_attr {
Index: openib_gen2/drivers/infiniband/hw/mthca/mthca_provider.c
===================================================================
--- openib_gen2.orig/drivers/infiniband/hw/mthca/mthca_provider.c
+++ openib_gen2/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -1270,7 +1270,10 @@ int mthca_register_device(struct mthca_d
(1ull << IB_USER_VERBS_CMD_DETACH_MCAST) |
(1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
(1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
- (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ);
+ (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) |
+ (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
+ (1ull << IB_USER_VERBS_CMD_QUERY_SRQ);
+
dev->ib_dev.node_type = RDMA_NODE_IB_CA;
dev->ib_dev.phys_port_cnt = dev->limits.num_ports;
dev->ib_dev.dma_device = &dev->pdev->dev;
@@ -1291,7 +1294,8 @@ int mthca_register_device(struct mthca_d
if (dev->mthca_flags & MTHCA_FLAG_SRQ) {
dev->ib_dev.create_srq = mthca_create_srq;
- dev->ib_dev.modify_srq = mthca_modify_srq;
+ dev->ib_dev.modify_srq = mthca_modify_srq;
+ dev->ib_dev.query_srq = mthca_query_srq;
dev->ib_dev.destroy_srq = mthca_destroy_srq;
if (mthca_is_memfree(dev))
@@ -1302,6 +1306,7 @@ int mthca_register_device(struct mthca_d
dev->ib_dev.create_qp = mthca_create_qp;
dev->ib_dev.modify_qp = mthca_modify_qp;
+ dev->ib_dev.query_qp = mthca_query_qp;
dev->ib_dev.destroy_qp = mthca_destroy_qp;
dev->ib_dev.create_cq = mthca_create_cq;
dev->ib_dev.resize_cq = mthca_resize_cq;
Index: openib_gen2/drivers/infiniband/hw/mthca/mthca_qp.c
===================================================================
--- openib_gen2.orig/drivers/infiniband/hw/mthca/mthca_qp.c
+++ openib_gen2/drivers/infiniband/hw/mthca/mthca_qp.c
@@ -270,6 +270,32 @@ static int to_mthca_state(enum ib_qp_sta
}
}
+
+static inline enum ib_qp_state to_ib_qp_state(int mthca_state)
+{
+ switch (mthca_state) {
+ case MTHCA_QP_STATE_RST: return IB_QPS_RESET;
+ case MTHCA_QP_STATE_INIT: return IB_QPS_INIT;
+ case MTHCA_QP_STATE_RTR: return IB_QPS_RTR;
+ case MTHCA_QP_STATE_RTS: return IB_QPS_RTS;
+ case MTHCA_QP_STATE_DRAINING:
+ case MTHCA_QP_STATE_SQD: return IB_QPS_SQD;
+ case MTHCA_QP_STATE_SQE: return IB_QPS_SQE;
+ case MTHCA_QP_STATE_ERR: return IB_QPS_ERR;
+ default: return -1;
+ }
+}
+
+static inline enum ib_mig_state to_ib_mig_state(int mthca_mig_state)
+{
+ switch (mthca_mig_state) {
+ case 0: return IB_MIG_ARMED;
+ case 1: return IB_MIG_REARM;
+ case 3: return IB_MIG_MIGRATED;
+ default: BUG();
+ }
+}
+
enum { RC, UC, UD, RD, RDEE, MLX, NUM_TRANS };
static int to_mthca_st(int transport)
@@ -553,6 +579,44 @@ static __be32 get_hw_access_flags(struct
return cpu_to_be32(hw_access_flags);
}
+static int to_ib_qp_access_flags(int mthca_flags)
+{
+ int ib_flags = 0;
+
+ if (mthca_flags & MTHCA_QP_BIT_RRE)
+ ib_flags |= IB_ACCESS_REMOTE_READ;
+
+ if (mthca_flags & MTHCA_QP_BIT_RWE)
+ ib_flags |= IB_ACCESS_REMOTE_WRITE;
+
+ if (mthca_flags & MTHCA_QP_BIT_RAE)
+ ib_flags |= IB_ACCESS_REMOTE_ATOMIC;
+
+ return ib_flags;
+}
+
+static void to_ib_ah_attr(struct mthca_dev *dev, struct ib_ah_attr *ib_ah_attr,
+ struct mthca_qp_path *path)
+{
+ memset(ib_ah_attr, 0, sizeof *path);
+ ib_ah_attr->port_num = (be32_to_cpu(path->port_pkey) >> 24) & 0x3;
+ ib_ah_attr->dlid = be16_to_cpu(path->rlid);
+ ib_ah_attr->sl = be32_to_cpu(path->sl_tclass_flowlabel) >> 28;
+ ib_ah_attr->src_path_bits = path->g_mylmc & 0x7f;
+ ib_ah_attr->static_rate = path->static_rate & 0x7;
+ ib_ah_attr->ah_flags = (path->g_mylmc & (1 << 7)) ? IB_AH_GRH : 0;
+ if (ib_ah_attr->ah_flags) {
+ ib_ah_attr->grh.sgid_index = path->mgid_index & (dev->limits.gid_table_len - 1);
+ ib_ah_attr->grh.hop_limit = path->hop_limit;
+ ib_ah_attr->grh.traffic_class =
+ (be32_to_cpu(path->sl_tclass_flowlabel) >> 20) & 0xff;
+ ib_ah_attr->grh.flow_label =
+ be32_to_cpu(path->sl_tclass_flowlabel) & 0xfffff;
+ memcpy(ib_ah_attr->grh.dgid.raw,
+ path->rgid, sizeof ib_ah_attr->grh.dgid.raw);
+ }
+}
+
static void mthca_path_set(struct ib_ah_attr *ah, struct mthca_qp_path *path)
{
path->g_mylmc = ah->src_path_bits & 0x7f;
@@ -914,6 +978,79 @@ int mthca_modify_qp(struct ib_qp *ibqp,
return err;
}
+
+
+int mthca_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
+ struct ib_qp_init_attr *qp_init_attr)
+{
+ struct mthca_dev *dev = to_mdev(ibqp->device);
+ struct mthca_qp *qp = to_mqp(ibqp);
+ int err;
+ struct mthca_mailbox *mailbox;
+ struct mthca_qp_param *qp_param;
+ struct mthca_qp_context *context;
+ int mthca_state;
+ u8 status;
+
+ mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
+ if (IS_ERR(mailbox))
+ return PTR_ERR(mailbox);
+
+ err = mthca_QUERY_QP(dev, qp->qpn, 0, mailbox, &status);
+ if (err)
+ goto exit;
+
+ qp_param = mailbox->buf;
+ context = &qp_param->context;
+ mthca_state = be32_to_cpu(context->flags) >> 28;
+ qp_attr->qp_state = to_ib_qp_state(mthca_state);
+ qp_attr->cur_qp_state = qp_attr->qp_state;
+ qp_attr->path_mtu = context->mtu_msgmax >> 5;
+ qp_attr->path_mig_state =
+ to_ib_mig_state((be32_to_cpu(context->flags) >> 11) & 0x3);
+ qp_attr->qkey = be32_to_cpu(context->qkey);
+ qp_attr->rq_psn = be32_to_cpu(context->rnr_nextrecvpsn) & 0xffffff;
+ qp_attr->sq_psn = be32_to_cpu(context->next_send_psn) & 0xffffff;
+ qp_attr->dest_qp_num = be32_to_cpu(context->remote_qpn) & 0xffffff;
+ qp_attr->qp_access_flags =
+ to_ib_qp_access_flags(be32_to_cpu(context->params2));
+ qp_attr->cap.max_send_wr = qp->sq.max;
+ qp_attr->cap.max_recv_wr = qp->rq.max;
+ qp_attr->cap.max_send_sge = qp->sq.max_gs;
+ qp_attr->cap.max_recv_sge = qp->rq.max_gs;
+ qp_attr->cap.max_inline_data = qp->max_inline_data;
+
+ to_ib_ah_attr(dev, &qp_attr->ah_attr, &context->pri_path);
+ to_ib_ah_attr(dev, &qp_attr->alt_ah_attr, &context->alt_path);
+
+ qp_attr->pkey_index = be32_to_cpu(context->pri_path.port_pkey) & 0x7f;
+ qp_attr->alt_pkey_index = be32_to_cpu(context->alt_path.port_pkey) & 0x7f;
+
+
+ /* qp_attr->en_sqd_async_notify
+ this field is only applicable in modify qp */
+ qp_attr->sq_draining = mthca_state == MTHCA_QP_STATE_DRAINING;
+
+ qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context->params1) >> 21) & 0x7);
+
+ qp_attr->max_dest_rd_atomic =
+ 1 << ((be32_to_cpu(context->params2) >> 21) & 0x7);
+ qp_attr->min_rnr_timer =
+ (be32_to_cpu(context->rnr_nextrecvpsn) >> 24) & 0x1f;
+ qp_attr->port_num = qp_attr->ah_attr.port_num;
+ qp_attr->timeout = context->pri_path.ackto >> 3;
+ qp_attr->retry_cnt = (be32_to_cpu(context->params1) >> 16) & 0x7;
+ qp_attr->rnr_retry = context->pri_path.rnr_retry >> 5;
+ qp_attr->alt_port_num = qp_attr->alt_ah_attr.port_num;
+ qp_attr->alt_timeout = context->alt_path.ackto >> 3;
+ qp_init_attr->cap = qp_attr->cap;
+
+exit:
+ mthca_free_mailbox(dev, mailbox);
+ return err;
+}
+
+
static int mthca_max_data_size(struct mthca_dev *dev, struct mthca_qp *qp, int desc_sz)
{
/*
Index: openib_gen2/drivers/infiniband/hw/mthca/mthca_cmd.c
===================================================================
--- openib_gen2.orig/drivers/infiniband/hw/mthca/mthca_cmd.c
+++ openib_gen2/drivers/infiniband/hw/mthca/mthca_cmd.c
@@ -1554,6 +1554,15 @@ int mthca_SW2HW_SRQ(struct mthca_dev *de
CMD_TIME_CLASS_A, status);
}
+
+int mthca_QUERY_SRQ(struct mthca_dev *dev, u32 num,
+ struct mthca_mailbox *mailbox, u8 *status)
+{
+
+ return mthca_cmd_box(dev, 0, mailbox->dma, num, 0,
+ CMD_QUERY_SRQ, CMD_TIME_CLASS_A, status);
+}
+
int mthca_HW2SW_SRQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox,
int srq_num, u8 *status)
{
Index: openib_gen2/drivers/infiniband/hw/mthca/mthca_dev.h
===================================================================
--- openib_gen2.orig/drivers/infiniband/hw/mthca/mthca_dev.h
+++ openib_gen2/drivers/infiniband/hw/mthca/mthca_dev.h
@@ -486,6 +486,7 @@ int mthca_alloc_srq(struct mthca_dev *de
void mthca_free_srq(struct mthca_dev *dev, struct mthca_srq *srq);
int mthca_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
enum ib_srq_attr_mask attr_mask);
+int mthca_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
void mthca_srq_event(struct mthca_dev *dev, u32 srqn,
enum ib_event_type event_type);
void mthca_free_srq_wqe(struct mthca_srq *srq, u32 wqe_addr);
@@ -497,6 +498,10 @@ int mthca_arbel_post_srq_recv(struct ib_
void mthca_qp_event(struct mthca_dev *dev, u32 qpn,
enum ib_event_type event_type);
int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask);
+
+int mthca_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
+ struct ib_qp_init_attr *qp_init_attr);
+
int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
struct ib_send_wr **bad_wr);
int mthca_tavor_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
More information about the general
mailing list