[ofa-general] enabling invalidate operations in libibverbs API
Jack Morgenstein
jackm at dev.mellanox.co.il
Wed Jun 11 01:37:22 PDT 2008
On Tuesday 10 June 2008 00:52, Roland Dreier wrote:
> Here's the complete (untested) libibverbs patch I am working on now... I
> think I got everything needed...
It looks OK to me. I'm not crazy about using an extra capability flag
(IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS), but I also don't see any way around this
if we are to preserve backwards binary compatibility.
For the same reason, I don't see any alternative regarding the unnamed union -- I
actually think that the unnamed union works well here.
- Jack
> diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
> index 0db083a..b503a28 100644
> --- a/include/infiniband/kern-abi.h
> +++ b/include/infiniband/kern-abi.h
> @@ -306,7 +306,10 @@ struct ibv_kern_wc {
> __u32 opcode;
> __u32 vendor_err;
> __u32 byte_len;
> - __u32 imm_data;
> + union {
> + __u32 imm_data;
> + __u32 invalidate_rkey;
> + };
> __u32 qp_num;
> __u32 src_qp;
> __u32 wc_flags;
> @@ -572,7 +575,10 @@ struct ibv_kern_send_wr {
> __u32 num_sge;
> __u32 opcode;
> __u32 send_flags;
> - __u32 imm_data;
> + union {
> + __u32 imm_data;
> + __u32 invalidate_rkey;
> + };
> union {
> struct {
> __u64 remote_addr;
> diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
> index a04cc62..e6e2b10 100644
> --- a/include/infiniband/verbs.h
> +++ b/include/infiniband/verbs.h
> @@ -92,7 +92,17 @@ enum ibv_device_cap_flags {
> IBV_DEVICE_SYS_IMAGE_GUID = 1 << 11,
> IBV_DEVICE_RC_RNR_NAK_GEN = 1 << 12,
> IBV_DEVICE_SRQ_RESIZE = 1 << 13,
> - IBV_DEVICE_N_NOTIFY_CQ = 1 << 14
> + IBV_DEVICE_N_NOTIFY_CQ = 1 << 14,
> + /*
> + * IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS is used by libibverbs to
> + * signal to low-level driver libraries that the kernel set
> + * the "send with invalidate" capaibility bit. Applications
> + * should only test IBV_DEVICE_MEM_MGT_EXTENSIONS and never
> + * look at IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS.
> + */
> + IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS = 1 << 16,
> + IBV_DEVICE_MEM_WINDOW = 1 << 17,
> + IBV_DEVICE_MEM_MGT_EXTENSIONS = 1 << 21,
> };
>
> enum ibv_atomic_cap {
> @@ -257,7 +267,8 @@ enum ibv_wc_opcode {
>
> enum ibv_wc_flags {
> IBV_WC_GRH = 1 << 0,
> - IBV_WC_WITH_IMM = 1 << 1
> + IBV_WC_WITH_IMM = 1 << 1,
> + IBV_WC_WITH_INVALIDATE = 1 << 2,
> };
>
> struct ibv_wc {
> @@ -266,7 +277,10 @@ struct ibv_wc {
> enum ibv_wc_opcode opcode;
> uint32_t vendor_err;
> uint32_t byte_len;
> - uint32_t imm_data; /* in network byte order */
> + union {
> + uint32_t imm_data; /* in network byte order */
> + uint32_t invalidate_rkey;
> + };
> uint32_t qp_num;
> uint32_t src_qp;
> enum ibv_wc_flags wc_flags;
> @@ -486,7 +500,11 @@ enum ibv_wr_opcode {
> IBV_WR_SEND_WITH_IMM,
> IBV_WR_RDMA_READ,
> IBV_WR_ATOMIC_CMP_AND_SWP,
> - IBV_WR_ATOMIC_FETCH_AND_ADD
> + IBV_WR_ATOMIC_FETCH_AND_ADD,
> + IBV_WR_LSO,
> + IBV_WR_SEND_WITH_INV,
> + IBV_WR_RDMA_READ_WITH_INV,
> + IBV_WR_LOCAL_INV,
> };
>
> enum ibv_send_flags {
> @@ -509,7 +527,10 @@ struct ibv_send_wr {
> int num_sge;
> enum ibv_wr_opcode opcode;
> enum ibv_send_flags send_flags;
> - uint32_t imm_data; /* in network byte order */
> + union {
> + uint32_t imm_data; /* in network byte order */
> + uint32_t invalidate_rkey;
> + };
> union {
> struct {
> uint64_t remote_addr;
> diff --git a/src/cmd.c b/src/cmd.c
> index 66d7134..1945143 100644
> --- a/src/cmd.c
> +++ b/src/cmd.c
> @@ -159,6 +159,18 @@ int ibv_cmd_query_device(struct ibv_context *context,
> device_attr->local_ca_ack_delay = resp.local_ca_ack_delay;
> device_attr->phys_port_cnt = resp.phys_port_cnt;
>
> + /*
> + * If the kernel driver says that it supports memory
> + * management extensions, then move the flag to
> + * IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS so that the low-level
> + * driver needs to move the flag back to show it supports the
> + * operations as well.
> + */
> + if (device_attr->device_cap_flags & IBV_DEVICE_MEM_MGT_EXTENSIONS) {
> + device_attr->device_cap_flags &= ~IBV_DEVICE_MEM_MGT_EXTENSIONS;
> + device_attr->device_cap_flags |= IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS;
> + }
> +
> return 0;
> }
>
> diff --git a/src/compat-1_0.c b/src/compat-1_0.c
> index 459ade9..0df8b68 100644
> --- a/src/compat-1_0.c
> +++ b/src/compat-1_0.c
> @@ -535,7 +535,18 @@ symver(__ibv_ack_async_event_1_0, ibv_ack_async_event, IBVERBS_1.0);
> int __ibv_query_device_1_0(struct ibv_context_1_0 *context,
> struct ibv_device_attr *device_attr)
> {
> - return ibv_query_device(context->real_context, device_attr);
> + int ret;
> +
> + ret = ibv_query_device(context->real_context, device_attr);
> +
> + /*
> + * ABI 1.0 consumers are never expecting memory management
> + * extension support.
> + */
> + device_attr->device_cap_flags &= ~(IBV_DEVICE_MEM_MGT_EXTENSIONS |
> + IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS);
> +
> + return ret;
> }
> symver(__ibv_query_device_1_0, ibv_query_device, IBVERBS_1.0);
>
> diff --git a/src/verbs.c b/src/verbs.c
> index 9e370ce..4ea342f 100644
> --- a/src/verbs.c
> +++ b/src/verbs.c
> @@ -79,7 +79,13 @@ enum ibv_rate mult_to_ibv_rate(int mult)
> int __ibv_query_device(struct ibv_context *context,
> struct ibv_device_attr *device_attr)
> {
> - return context->ops.query_device(context, device_attr);
> + int ret;
> +
> + ret = context->ops.query_device(context, device_attr);
> +
> + device_attr->device_cap_flags &= ~IBV_DEVICE_KERN_MEM_MGT_EXTENSIONS;
> +
> + return ret;
> }
> default_symver(__ibv_query_device, ibv_query_device);
>
> _______________________________________________
> general mailing list
> general at lists.openfabrics.org
> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
>
> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
>
More information about the general
mailing list