[openib-general] [PATCH 13/18] [RFC] RDMA Core types/fastpath poll
Steve Wise
swise at opengridcomputing.com
Mon Mar 6 10:09:10 PST 2006
--- old/src/linux-kernel/infiniband/hw/cxgb3/core/cxio_wr.h 1969-12-31 18:00:00.000000000 -0600
+++ new/src/linux-kernel/infiniband/hw/cxgb3/core/cxio_wr.h 2006-03-06 09:26:21.000000000 -0600
@@ -0,0 +1,600 @@
+/*
+ * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
+ * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef __CXIO_WR_H__
+#define __CXIO_WR_H__
+
+#include <asm/io.h>
+#include <linux/pci.h>
+#include "firmware_exports.h"
+
+#define T3_MAX_SGE 4
+
+#define Q_EMPTY(rptr,wptr) ((rptr)==(wptr))
+#define Q_FULL(rptr,wptr,size_log2) ( (((wptr)-(rptr))>>(size_log2)) && ((rptr)!=(wptr)) )
+#define Q_GENBIT(ptr,size_log2) (!(((ptr)>>size_log2)&0x1))
+#define Q_FREECNT(rptr,wptr,size_log2) ((1UL<<size_log2)-((wptr)-(rptr)))
+#define Q_COUNT(rptr,wptr) ((wptr)-(rptr))
+#define Q_PTR2IDX(ptr,size_log2) (ptr & ((1UL<<size_log2)-1))
+#define RING_DOORBELL(doorbell, QPID) { \
+ (writel(((1<<31) | (QPID)),doorbell)); \
+}
+
+#define SEQ32_GE(x,y) (!( (((u32) (x)) - ((u32) (y))) & 0x80000000 ))
+
+enum t3_wr_flags {
+ T3_COMPLETION_FLAG = 0x01,
+ T3_NOTIFY_FLAG = 0x02,
+ T3_SOLICITED_EVENT_FLAG = 0x04,
+ T3_READ_FENCE_FLAG = 0x08,
+ T3_LOCAL_FENCE_FLAG = 0x10
+} __attribute__ ((packed));
+
+enum t3_wr_opcode {
+ T3_WR_BP = FW_WROPCODE_RI_BYPASS,
+ T3_WR_SEND = FW_WROPCODE_RI_SEND,
+ T3_WR_WRITE = FW_WROPCODE_RI_RDMA_WRITE,
+ T3_WR_READ = FW_WROPCODE_RI_RDMA_READ,
+ T3_WR_INV_STAG = FW_WROPCODE_RI_LOCAL_INV,
+ T3_WR_BIND = FW_WROPCODE_RI_BIND_MW,
+ T3_WR_RCV = FW_WROPCODE_RI_RECEIVE,
+ T3_WR_INIT = FW_WROPCODE_RI_RDMA_INIT,
+ T3_WR_QP_MOD = FW_WROPCODE_RI_MODIFY_QP
+} __attribute__ ((packed));
+
+enum t3_rdma_opcode {
+ T3_RDMA_WRITE, /* IETF RDMAP v1.0 ... */
+ T3_READ_REQ,
+ T3_READ_RESP,
+ T3_SEND,
+ T3_SEND_WITH_INV,
+ T3_SEND_WITH_SE,
+ T3_SEND_WITH_SE_INV,
+ T3_TERMINATE,
+ T3_RDMA_INIT, /* CHELSIO RI specific ... */
+ T3_BIND_MW,
+ T3_FAST_REGISTER,
+ T3_LOCAL_INV,
+ T3_QP_MOD,
+ T3_BYPASS
+} __attribute__ ((packed));
+
+/* Work request id */
+union t3_wrid {
+ struct {
+ u32 hi:32;
+ u32 low:32;
+ } id0;
+ u64 id1;
+};
+
+#define WRID(wrid) (wrid.id1)
+#define WRID_GEN(wrid) (wrid.id0.wr_gen)
+#define WRID_IDX(wrid) (wrid.id0.wr_idx)
+#define WRID_LO(wrid) (wrid.id0.wr_lo)
+
+struct fw_riwrh {
+ u32 op_seop_flags;
+ u32 gen_tid_len;
+};
+
+#define S_FW_RIWR_OP 24
+#define V_FW_RIWR_OP(x) ((x) << S_FW_RIWR_OP)
+
+#define S_FW_RIWR_SOPEOP 22
+#define M_FW_RIWR_SOPEOP 0x3
+#define V_FW_RIWR_SOPEOP(x) ((x) << S_FW_RIWR_SOPEOP)
+
+#define S_FW_RIWR_FLAGS 8
+#define V_FW_RIWR_FLAGS(x) ((x) << S_FW_RIWR_FLAGS)
+
+#define S_FW_RIWR_TID 8
+#define V_FW_RIWR_TID(x) ((x) << S_FW_RIWR_TID)
+
+#define S_FW_RIWR_LEN 0
+#define V_FW_RIWR_LEN(x) ((x) << S_FW_RIWR_LEN)
+
+#define S_FW_RIWR_GEN 31
+#define V_FW_RIWR_GEN(x) ((x) << S_FW_RIWR_GEN)
+
+struct t3_sge {
+ u32 stag;
+ u32 len;
+ u64 to;
+};
+
+/* If num_sgle is zero, flit 5+ contains immediate data.*/
+struct t3_send_wr {
+ struct fw_riwrh wrh; /* 0 */
+ union t3_wrid wrid; /* 1 */
+
+ enum t3_rdma_opcode rdmaop:8;
+ u32 reserved:24; /* 2 */
+ u32 rem_stag; /* 2 */
+ u32 plen; /* 3 */
+ u32 num_sgle;
+ struct t3_sge sgl[T3_MAX_SGE]; /* 4+ */
+};
+
+struct t3_local_inv_wr {
+ struct fw_riwrh wrh; /* 0 */
+ union t3_wrid wrid; /* 1 */
+ u32 stag; /* 2 */
+ u32 reserved3;
+};
+
+struct t3_rdma_write_wr {
+ struct fw_riwrh wrh; /* 0 */
+ union t3_wrid wrid; /* 1 */
+ enum t3_rdma_opcode rdmaop:8; /* 2 */
+ u32 reserved:24; /* 2 */
+ u32 stag_sink;
+ u64 to_sink; /* 3 */
+ u32 plen; /* 4 */
+ u32 num_sgle;
+ struct t3_sge sgl[T3_MAX_SGE]; /* 5+ */
+};
+
+struct t3_rdma_read_wr {
+ struct fw_riwrh wrh; /* 0 */
+ union t3_wrid wrid; /* 1 */
+ enum t3_rdma_opcode rdmaop:8; /* 2 */
+ u32 reserved:24;
+ u32 rem_stag;
+ u64 rem_to; /* 3 */
+ u32 local_stag; /* 4 */
+ u32 local_len;
+ u64 local_to; /* 5 */
+};
+
+enum t3_addr_type {
+ T3_VA_BASED_TO = 0x0,
+ T3_ZERO_BASED_TO = 0x1
+} __attribute__ ((packed));
+
+enum t3_mem_perms {
+ T3_MEM_ACCESS_LOCAL_READ = 0x1,
+ T3_MEM_ACCESS_LOCAL_WRITE = 0x2,
+ T3_MEM_ACCESS_REM_READ = 0x4,
+ T3_MEM_ACCESS_REM_WRITE = 0x8
+} __attribute__ ((packed));
+
+struct t3_bind_mw_wr {
+ struct fw_riwrh wrh; /* 0 */
+ union t3_wrid wrid; /* 1 */
+ u32 reserved:16;
+ enum t3_addr_type type:8;
+ enum t3_mem_perms perms:8; /* 2 */
+ u32 mr_stag;
+ u32 mw_stag; /* 3 */
+ u32 mw_len;
+ u64 mw_va; /* 4 */
+ u32 mr_pbl_addr; /* 5 */
+ u32 reserved2:24;
+ u32 mr_pagesz:8;
+};
+
+struct t3_receive_wr {
+ struct fw_riwrh wrh; /* 0 */
+ union t3_wrid wrid; /* 1 */
+ u8 pagesz[T3_MAX_SGE];
+ u32 num_sgle; /* 2 */
+ struct t3_sge sgl[T3_MAX_SGE]; /* 3+ */
+ u32 pbl_addr[T3_MAX_SGE];
+};
+
+struct t3_bypass_wr {
+ struct fw_riwrh wrh;
+ union t3_wrid wrid; /* 1 */
+};
+
+struct t3_modify_qp_wr {
+ struct fw_riwrh wrh;
+ union t3_wrid wrid;
+ u64 ctx1;
+ u64 ctx0;
+};
+
+enum t3_mpa_attrs {
+ uP_RI_MPA_RX_MARKER_ENABLE = 0x1,
+ uP_RI_MPA_TX_MARKER_ENABLE = 0x2,
+ uP_RI_MPA_CRC_ENABLE = 0x4,
+ uP_RI_MPA_IETF_ENABLE = 0x8
+} __attribute__ ((packed));
+
+enum t3_qp_caps {
+ uP_RI_QP_RDMA_READ_ENABLE = 0x01,
+ uP_RI_QP_RDMA_WRITE_ENABLE = 0x02,
+ uP_RI_QP_BIND_ENABLE = 0x04,
+ uP_RI_QP_FAST_REGISTER_ENABLE = 0x08,
+ uP_RI_QP_STAG0_ENABLE = 0x10
+} __attribute__ ((packed));
+
+struct t3_rdma_init_attr {
+ u32 tid;
+ u32 qpid;
+ u32 pdid;
+ u32 scqid;
+ u32 rcqid;
+ u32 rq_addr;
+ u32 rq_size;
+ enum t3_mpa_attrs mpaattrs;
+ enum t3_qp_caps qpcaps;
+ u16 tcp_emss;
+ u32 ord;
+ u32 ird;
+ u64 qp_dma_addr;
+ u32 qp_dma_size;
+};
+
+struct t3_rdma_init_wr {
+ struct fw_riwrh wrh; /* 9 */
+ union t3_wrid wrid; /* 1 */
+ u32 qpid; /* 2 */
+ u32 pdid;
+ u32 scqid; /* 3 */
+ u32 rcqid;
+ u32 rq_addr; /* 4 */
+ u32 rq_size;
+ enum t3_mpa_attrs mpaattrs:8; /* 5 */
+ enum t3_qp_caps qpcaps:8;
+ u32 ulpdu_size:16;
+ u32 reserved3;
+ u32 ord; /* 6 */
+ u32 ird;
+ u64 qp_dma_addr; /* 7 */
+ u32 qp_dma_size; /* 8 */
+ u32 rsvd;
+};
+
+union t3_wr {
+ struct t3_send_wr send;
+ struct t3_rdma_write_wr write;
+ struct t3_rdma_read_wr read;
+ struct t3_receive_wr recv;
+ struct t3_local_inv_wr local_inv;
+ struct t3_bind_mw_wr bind;
+ struct t3_bypass_wr bypass;
+ struct t3_rdma_init_wr init;
+ struct t3_modify_qp_wr qp_mod;
+ u64 flit[16];
+};
+
+#define T3_SQ_COOKIE_FLIT 14
+#define T3_RQ_COOKIE_FLIT 13
+
+static inline void build_fw_riwrh(struct fw_riwrh *wqe, enum t3_wr_opcode op,
+ enum t3_wr_flags flags, u8 genbit, u32 tid,
+ u8 len)
+{
+ wqe->op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(op) |
+ V_FW_RIWR_SOPEOP(M_FW_RIWR_SOPEOP) |
+ V_FW_RIWR_FLAGS(flags));
+ wmb();
+ wqe->gen_tid_len = cpu_to_be32(V_FW_RIWR_GEN(genbit) |
+ V_FW_RIWR_TID(tid) |
+ V_FW_RIWR_LEN(len));
+ /* 2nd gen bit... */
+ ((union t3_wr *)wqe)->flit[15] = cpu_to_be64(genbit);
+}
+
+/*
+ * T3 ULP2_TX commands
+ */
+enum t3_utx_mem_op {
+ T3_UTX_MEM_READ = 2,
+ T3_UTX_MEM_WRITE = 3
+};
+
+/* T3 MC7 RDMA TPT entry format */
+
+enum tpt_mem_type {
+ TPT_NON_SHARED_MR = 0x0,
+ TPT_SHARED_MR = 0x1,
+ TPT_MW = 0x2,
+ TPT_MW_RELAXED_PROTECTION = 0x3
+};
+
+enum tpt_addr_type {
+ TPT_ZBTO = 0,
+ TPT_VATO = 1
+};
+
+enum tpt_mem_perm {
+ TPT_LOCAL_READ = 0x8,
+ TPT_LOCAL_WRITE = 0x4,
+ TPT_REMOTE_READ = 0x2,
+ TPT_REMOTE_WRITE = 0x1
+};
+
+struct tpt_entry {
+ u32 valid_stag_pdid;
+ u32 flags_pagesize_qpid;
+
+ u32 rsvd_pbl_addr;
+ u32 len;
+ u32 va_hi;
+ u32 va_low_or_fbo;
+
+ u32 rsvd_bind_cnt_or_pstag;
+ u32 rsvd_pbl_size;
+};
+#define S_TPT_VALID 31
+#define V_TPT_VALID(x) ((x) << S_TPT_VALID)
+#define F_TPT_VALID V_TPT_VALID(1U)
+
+#define S_TPT_STAG_KEY 23
+#define M_TPT_STAG_KEY 0xFF
+#define V_TPT_STAG_KEY(x) ((x) << S_TPT_STAG_KEY)
+#define G_TPT_STAG_KEY(x) (((x) >> S_TPT_STAG_KEY) & M_TPT_STAG_KEY)
+
+#define S_TPT_STAG_STATE 22
+#define V_TPT_STAG_STATE(x) ((x) << S_TPT_STAG_STATE)
+#define F_TPT_STAG_STATE V_TPT_STAG_STATE(1U)
+
+#define S_TPT_STAG_TYPE 20
+#define M_TPT_STAG_TYPE 0x3
+#define V_TPT_STAG_TYPE(x) ((x) << S_TPT_STAG_TYPE)
+#define G_TPT_STAG_TYPE(x) (((x) >> S_TPT_STAG_TYPE) & M_TPT_STAG_TYPE)
+
+#define S_TPT_PDID 0
+#define M_TPT_PDID 0xFFFFF
+#define V_TPT_PDID(x) ((x) << S_TPT_PDID)
+#define G_TPT_PDID(x) (((x) >> S_TPT_PDID) & M_TPT_PDID)
+
+#define S_TPT_PERM 28
+#define M_TPT_PERM 0xF
+#define V_TPT_PERM(x) ((x) << S_TPT_PERM)
+#define G_TPT_PERM(x) (((x) >> S_TPT_PERM) & M_TPT_PERM)
+
+#define S_TPT_REM_INV_DIS 27
+#define V_TPT_REM_INV_DIS(x) ((x) << S_TPT_REM_INV_DIS)
+#define F_TPT_REM_INV_DIS V_TPT_REM_INV_DIS(1U)
+
+#define S_TPT_ADDR_TYPE 26
+#define V_TPT_ADDR_TYPE(x) ((x) << S_TPT_ADDR_TYPE)
+#define F_TPT_ADDR_TYPE V_TPT_ADDR_TYPE(1U)
+
+#define S_TPT_MW_BIND_ENABLE 25
+#define V_TPT_MW_BIND_ENABLE(x) ((x) << S_TPT_MW_BIND_ENABLE)
+#define F_TPT_MW_BIND_ENABLE V_TPT_MW_BIND_ENABLE(1U)
+
+#define S_TPT_PAGE_SIZE 20
+#define M_TPT_PAGE_SIZE 0x1F
+#define V_TPT_PAGE_SIZE(x) ((x) << S_TPT_PAGE_SIZE)
+#define G_TPT_PAGE_SIZE(x) (((x) >> S_TPT_PAGE_SIZE) & M_TPT_PAGE_SIZE)
+
+#define S_TPT_PBL_ADDR 0
+#define M_TPT_PBL_ADDR 0x1FFFFFFF
+#define V_TPT_PBL_ADDR(x) ((x) << S_TPT_PBL_ADDR)
+#define G_TPT_PBL_ADDR(x) (((x) >> S_TPT_PBL_ADDR) & M_TPT_PBL_ADDR)
+
+#define S_TPT_QPID 0
+#define M_TPT_QPID 0xFFFFF
+#define V_TPT_QPID(x) ((x) << S_TPT_QPID)
+#define G_TPT_QPID(x) (((x) >> S_TPT_QPID) & M_TPT_QPID)
+
+#define S_TPT_PSTAG 0
+#define M_TPT_PSTAG 0xFFFFFF
+#define V_TPT_PSTAG(x) ((x) << S_TPT_PSTAG)
+#define G_TPT_PSTAG(x) (((x) >> S_TPT_PSTAG) & M_TPT_PSTAG)
+
+#define S_TPT_PBL_SIZE 0
+#define M_TPT_PBL_SIZE 0xFFFFF
+#define V_TPT_PBL_SIZE(x) ((x) << S_TPT_PBL_SIZE)
+#define G_TPT_PBL_SIZE(x) (((x) >> S_TPT_PBL_SIZE) & M_TPT_PBL_SIZE)
+
+/*
+ * CQE defs
+ */
+struct t3_cqe {
+ u32 header:32;
+ u32 len:32;
+ u32 wrid_hi_stag:32;
+ u32 wrid_low_msn:32;
+};
+
+#define S_CQE_QPID 12
+#define M_CQE_QPID 0xFFFFF
+#define G_CQE_QPID(x) ((((x) >> S_CQE_QPID)) & M_CQE_QPID)
+
+#define S_CQE_GENBIT 10
+#define M_CQE_GENBIT 0x1
+#define G_CQE_GENBIT(x) (((x) >> S_CQE_GENBIT) & M_CQE_GENBIT)
+
+#define S_CQE_STATUS 5
+#define M_CQE_STATUS 0x1F
+#define G_CQE_STATUS(x) ((((x) >> S_CQE_STATUS)) & M_CQE_STATUS)
+#define V_CQE_STATUS(x) ((x)<<S_CQE_STATUS)
+
+#define S_CQE_TYPE 4
+#define M_CQE_TYPE 0x1
+#define G_CQE_TYPE(x) ((((x) >> S_CQE_TYPE)) & M_CQE_TYPE)
+
+#define S_CQE_OPCODE 0
+#define M_CQE_OPCODE 0xF
+#define G_CQE_OPCODE(x) ((((x) >> S_CQE_OPCODE)) & M_CQE_OPCODE)
+
+#define CQE_QPID(x) (G_CQE_QPID(be32_to_cpu((x).header)))
+#define CQE_GENBIT(x) (G_CQE_GENBIT(be32_to_cpu((x).header)))
+#define CQE_TYPE(x) (G_CQE_TYPE(be32_to_cpu((x).header)))
+#define SQ_TYPE(x) (CQE_TYPE((x)))
+#define RQ_TYPE(x) (!CQE_TYPE((x)))
+#define CQE_STATUS(x) (G_CQE_STATUS(be32_to_cpu((x).header)))
+#define CQE_OPCODE(x) (G_CQE_OPCODE(be32_to_cpu((x).header)))
+
+#define CQE_LEN(x) (be32_to_cpu((x).len))
+
+#define CQE_WRID_HI(x) (be32_to_cpu((x).wrid_hi_stag))
+#define CQE_WRID_LOW(x) (be32_to_cpu((x).wrid_low_msn))
+
+#define CQE_WRID_STAG(x) (be32_to_cpu((x).wrid_hi_stag))
+#define CQE_WRID_MSN(x) (be32_to_cpu((x).wrid_low_msn))
+
+#define TPT_ERR_SUCCESS 0x0 // success, no error detected.
+#define TPT_ERR_STAG 0x1 // STAG invalid : either the STAG is offlimt, being 0, or STAG_key mismatch
+#define TPT_ERR_PDID 0x2 // PDID mismatch
+#define TPT_ERR_QPID 0x3 // QPID mismatch
+#define TPT_ERR_ACCESS 0x4 // Invalid access right
+#define TPT_ERR_WRAP 0x5 // Wrap error
+#define TPT_ERR_BOUND 0x6 // base and bounds voilation
+#define TPT_ERR_INVALIDATE_SHARED_MR 0x7 // attempt to invalidate a shared memory region
+#define TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND 0x8 // attempt to invalidate a shared memory region
+#define TPT_ERR_ECC 0x9 // ECC error detected
+#define TPT_ERR_ECC_PSTAG 0xA // ECC error detected when reading the PSTAG for a MW Invalidate
+#define TPT_ERR_PBL_ADDR_BOUND 0xB // pbl address out of bound : software error
+#define TPT_ERR_CRC 0x10 // CRC error
+#define TPT_ERR_MARKER 0x11 // Marker error
+#define TPT_ERR_PDU_LEN_ERR 0x12 // invalid PDU length
+#define TPT_ERR_OUT_OF_RQE 0x13 // out of RQE
+#define TPT_ERR_DDP_VERSION 0x14 // wrong DDP version
+#define TPT_ERR_RDMA_VERSION 0x15 // wrong RDMA version
+#define TPT_ERR_OPCODE 0x16 // invalid rdma opcode
+#define TPT_ERR_DDP_QUEUE_NUM 0x17 // invalid ddp queue number
+#define TPT_ERR_MSN 0x18 // MSN error
+#define TPT_ERR_TBIT 0x19 // tag bit not set correctly
+#define TPT_ERR_MO 0x1A // MO not zero for TERMIANTE and READ_REQ
+#define TPT_ERR_MSN_GAP 0x1B
+#define TPT_ERR_MSN_RANGE 0x1C
+#define TPT_ERR_IRD_OVERFLOW 0x1D
+#define TPT_ERR_RQE_ADDR_BOUND 0x1E // RQE address out of bound : software error
+#define TPT_ERR_INTERNAL_ERR 0x1F // internel error (opcode mismatch)
+
+/*
+ * A T3 WQ implements both the SQ and RQ.
+ */
+struct t3_wq {
+ u32 error; /* 1 once we go to ERROR */
+ u32 qpid;
+ u32 wptr; /* idx to next available WR slot */
+ u32 size_log2; /* total wq size */
+ u32 sq_wptr; /* used to track sq full */
+ u32 sq_rptr; /* oldest pending sq WR */
+ u32 sq_size_log2; /* sq size */
+ dma_addr_t dma_addr;
+ DECLARE_PCI_UNMAP_ADDR(mapping)
+ union t3_wr *queue;
+ u32 rq_wptr; /* used to track rq full */
+ u32 rq_rptr; /* oldest pending rq WR */
+ u32 rq_size_log2; /* rq size */
+ volatile u32 *doorbell;
+ struct t3_cqe read_cqe; /* cache of the next pending read CQE */
+};
+
+struct t3_cq {
+ u32 cqid;
+ u32 rptr;
+ u32 wptr;
+ u32 size_log2;
+ dma_addr_t dma_addr;
+ DECLARE_PCI_UNMAP_ADDR(mapping)
+ struct t3_cqe *queue;
+};
+
+#define CQ_VLD_ENTRY(ptr,size_log2,cqe) (Q_GENBIT(ptr,size_log2)==CQE_GENBIT(*cqe))
+
+int __cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq,
+ struct t3_cqe *cqe, u8 * cqe_flushed,
+ u64 * cookie, u32 * credit);
+
+/*
+ * Fastpath poll.
+ *
+ * Caller must:
+ * check the validity of the first CQE,
+ * supply the wq assicated with the qpid.
+ * credit: cq credit to return to sge.
+ * cqe_flushed: 1 iff the CQE is flushed.
+ * cqe: copy of the polled CQE.
+ *
+ * return value:
+ * 0 CQE returned,
+ * -1 CQE skipped, try again.
+ */
+static inline int cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq,
+ struct t3_cqe *cqe, u8 * cqe_flushed,
+ u64 * cookie, u32 * credit)
+{
+ struct t3_cqe *rd_cqe;
+
+ *cqe_flushed = 0;
+ *credit = 0;
+ rd_cqe = cq->queue + (Q_PTR2IDX(cq->rptr, cq->size_log2));
+
+ /* fastpath:
+ * wq is valid
+ * wq not in error
+ * cqe status not error
+ * opcode not TERMINATE
+ * opcode not read response
+ * opcode not read request
+ * wq->read_cqe.header is 0.
+ */
+ if (likely(wq && !wq->error &&
+ !CQE_STATUS(*rd_cqe) &&
+ (CQE_OPCODE(*rd_cqe) != T3_TERMINATE) &&
+ (CQE_OPCODE(*rd_cqe) != T3_READ_RESP) &&
+ (CQE_OPCODE(*rd_cqe) != T3_READ_REQ) &&
+ !wq->read_cqe.header) &&
+ (SQ_TYPE(*rd_cqe) || (RQ_TYPE(*rd_cqe) &&
+ (CQE_WRID_MSN(*rd_cqe) ==
+ (wq->rq_rptr + 1))))) {
+ *cqe = *rd_cqe;
+
+ /*
+ * Reap the associated WR(s) that are freed up with this
+ * completion.
+ */
+ if (SQ_TYPE(*rd_cqe)) {
+ wq->sq_rptr = CQE_WRID_MSN(*rd_cqe);
+ *cookie = wq->queue[Q_PTR2IDX(wq->sq_rptr, wq->size_log2)].flit[T3_SQ_COOKIE_FLIT];
+ ++(wq->sq_rptr);
+ } else {
+ *cookie = wq->queue[Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2)].flit[T3_RQ_COOKIE_FLIT];
+ ++(wq->rq_rptr);
+ }
+
+ ++(cq->rptr);
+
+ /*
+ * compute credits.
+ */
+ if (((cq->rptr - cq->wptr) > (1 << (cq->size_log2 - 1)))
+ || ((cq->rptr - cq->wptr) >= 128)) {
+ *credit = cq->rptr - cq->wptr;
+ cq->wptr = cq->rptr;
+ }
+ return 0;
+ }
+ return __cxio_poll_cq(wq, cq, cqe, cqe_flushed, cookie, credit);
+}
+#endif
More information about the general
mailing list