[ofa-general] enabling invalidate operations in libibverbs API
Or Gerlitz
ogerlitz at voltaire.com
Thu Aug 28 03:43:25 PDT 2008
Roland Dreier wrote:
> Here's the complete (untested) libibverbs patch I am working on now... I think I got everything needed...
Hi Roland,
Where does this patch stands? specifically, with the integration of the
kernel part into 2.6.27
When discussed back on June 08, there were some minor fix you agreed
that need to be done here (remove the LSO bit)
Or.
>
> diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
> index 0db083a..b503a28 100644
> --- a/include/infiniband/kern-abi.h
> +++ b/include/infiniband/kern-abi.h
> @@ -306,7 +306,10 @@ struct ibv_kern_wc {
> __u32 opcode;
> __u32 vendor_err;
> __u32 byte_len;
> - __u32 imm_data;
> + union {
> + __u32 imm_data;
> + __u32 invalidate_rkey;
> + };
> __u32 qp_num;
> __u32 src_qp;
> __u32 wc_flags;
> @@ -572,7 +575,10 @@ struct ibv_kern_send_wr {
> __u32 num_sge;
> __u32 opcode;
> __u32 send_flags;
> - __u32 imm_data;
> + union {
> + __u32 imm_data;
> + __u32 invalidate_rkey;
> + };
> union {
> struct {
> __u64 remote_addr;
> diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
> index a04cc62..e6e2b10 100644
> --- a/include/infiniband/verbs.h
> +++ b/include/infiniband/verbs.h
> @@ -92,7 +92,17 @@ enum ibv_device_cap_flags {
> IBV_DEVICE_SYS_IMAGE_GUID = 1 << 11,
> IBV_DEVICE_RC_RNR_NAK_GEN = 1 << 12,
> IBV_DEVICE_SRQ_RESIZE = 1 << 13,
> - IBV_DEVICE_N_NOTIFY_CQ = 1 << 14
> + IBV_DEVICE_N_NOTIFY_CQ = 1 << 14,
> + /*
> + * IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS is used by libibverbs to
> + * signal to low-level driver libraries that the kernel set
> + * the "send with invalidate" capaibility bit. Applications
> + * should only test IBV_DEVICE_MEM_MGT_EXTENSIONS and never
> + * look at IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS.
> + */
> + IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS = 1 << 16,
> + IBV_DEVICE_MEM_WINDOW = 1 << 17,
> + IBV_DEVICE_MEM_MGT_EXTENSIONS = 1 << 21,
> };
>
> enum ibv_atomic_cap {
> @@ -257,7 +267,8 @@ enum ibv_wc_opcode {
>
> enum ibv_wc_flags {
> IBV_WC_GRH = 1 << 0,
> - IBV_WC_WITH_IMM = 1 << 1
> + IBV_WC_WITH_IMM = 1 << 1,
> + IBV_WC_WITH_INVALIDATE = 1 << 2,
> };
>
> struct ibv_wc {
> @@ -266,7 +277,10 @@ struct ibv_wc {
> enum ibv_wc_opcode opcode;
> uint32_t vendor_err;
> uint32_t byte_len;
> - uint32_t imm_data; /* in network byte order */
> + union {
> + uint32_t imm_data; /* in network byte order */
> + uint32_t invalidate_rkey;
> + };
> uint32_t qp_num;
> uint32_t src_qp;
> enum ibv_wc_flags wc_flags;
> @@ -486,7 +500,11 @@ enum ibv_wr_opcode {
> IBV_WR_SEND_WITH_IMM,
> IBV_WR_RDMA_READ,
> IBV_WR_ATOMIC_CMP_AND_SWP,
> - IBV_WR_ATOMIC_FETCH_AND_ADD
> + IBV_WR_ATOMIC_FETCH_AND_ADD,
> + IBV_WR_LSO,
> + IBV_WR_SEND_WITH_INV,
> + IBV_WR_RDMA_READ_WITH_INV,
> + IBV_WR_LOCAL_INV,
> };
>
> enum ibv_send_flags {
> @@ -509,7 +527,10 @@ struct ibv_send_wr {
> int num_sge;
> enum ibv_wr_opcode opcode;
> enum ibv_send_flags send_flags;
> - uint32_t imm_data; /* in network byte order */
> + union {
> + uint32_t imm_data; /* in network byte order */
> + uint32_t invalidate_rkey;
> + };
> union {
> struct {
> uint64_t remote_addr;
> diff --git a/src/cmd.c b/src/cmd.c
> index 66d7134..1945143 100644
> --- a/src/cmd.c
> +++ b/src/cmd.c
> @@ -159,6 +159,18 @@ int ibv_cmd_query_device(struct ibv_context *context,
> device_attr->local_ca_ack_delay = resp.local_ca_ack_delay;
> device_attr->phys_port_cnt = resp.phys_port_cnt;
>
> + /*
> + * If the kernel driver says that it supports memory
> + * management extensions, then move the flag to
> + * IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS so that the low-level
> + * driver needs to move the flag back to show it supports the
> + * operations as well.
> + */
> + if (device_attr->device_cap_flags & IBV_DEVICE_MEM_MGT_EXTENSIONS) {
> + device_attr->device_cap_flags &= ~IBV_DEVICE_MEM_MGT_EXTENSIONS;
> + device_attr->device_cap_flags |= IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS;
> + }
> +
> return 0;
> }
>
> diff --git a/src/compat-1_0.c b/src/compat-1_0.c
> index 459ade9..0df8b68 100644
> --- a/src/compat-1_0.c
> +++ b/src/compat-1_0.c
> @@ -535,7 +535,18 @@ symver(__ibv_ack_async_event_1_0, ibv_ack_async_event, IBVERBS_1.0);
> int __ibv_query_device_1_0(struct ibv_context_1_0 *context,
> struct ibv_device_attr *device_attr)
> {
> - return ibv_query_device(context->real_context, device_attr);
> + int ret;
> +
> + ret = ibv_query_device(context->real_context, device_attr);
> +
> + /*
> + * ABI 1.0 consumers are never expecting memory management
> + * extension support.
> + */
> + device_attr->device_cap_flags &= ~(IBV_DEVICE_MEM_MGT_EXTENSIONS |
> + IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS);
> +
> + return ret;
> }
> symver(__ibv_query_device_1_0, ibv_query_device, IBVERBS_1.0);
>
> diff --git a/src/verbs.c b/src/verbs.c
> index 9e370ce..4ea342f 100644
> --- a/src/verbs.c
> +++ b/src/verbs.c
> @@ -79,7 +79,13 @@ enum ibv_rate mult_to_ibv_rate(int mult)
> int __ibv_query_device(struct ibv_context *context,
> struct ibv_device_attr *device_attr)
> {
> - return context->ops.query_device(context, device_attr);
> + int ret;
> +
> + ret = context->ops.query_device(context, device_attr);
> +
> + device_attr->device_cap_flags &= ~IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS;
> +
> + return ret;
> }
> default_symver(__ibv_query_device, ibv_query_device);
>
> _______________________________________________
> general mailing list
> general at lists.openfabrics.org
> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
>
> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
More information about the general
mailing list