[ofa-general] [PATCH/RFC 1/2] IB/core: Add support for "send with invalidate" work requests
Roland Dreier
rdreier at cisco.com
Wed Mar 19 20:33:24 PDT 2008
Add an IB_SEND_INVALIDATE send flag that can be used to mark a "send
with invalidate" work request as defined in the iWARP verbs and the
InfiniBand base memory management extensions. Also add a new
"invalidate" structure to the struct ib_send_wr.wr union that can be
used to pass in an R_Key/STag to be invalidated. Add this new
structure to struct ib_uverbs_send_wr and add code to copy the rkey
field in ib_uverbs_post_send().
Also, move the existing IB_DEVICE_SEND_W_INV flag to a new bit, since
the iWARP drivers currently in the tree set the bit. The amso1100
driver at least will silently fail to honor the IB_SEND_INVALIDATE bit
if passed in as part of userspace send requests (since it does not
implement kernel bypass work request queueing). Remove the flag from
all existing drivers that set it until we know which ones are OK.
The values chosen for the new flags are not consecutive to avoid
clashing with flags defined in the IPoIB large send offload and XRC
patches, which are not merged yet but which are already in use and are
likely to be merged soon.
This resurrects a patch sent long ago by Mikkel Hagen <mhagen at iol.unh.edu>.
Signed-off-by: Roland Dreier <rolandd at cisco.com>
---
I tried to avoid clashes with any flags values defined in OFED 1.3,
just to make things a little simpler for users. However a double
check that I succeeded in avoiding already-used values would be good.
Also I think redefining the IB_DEVICE_SEND_W_INV to a new value avoids
all the backwards and forwards compatibility problems that were a
sticking point when this idea originally appeared.
Anyway if this looks good to people I will merge it for 2.6.26.
drivers/infiniband/core/uverbs_cmd.c | 4 ++++
drivers/infiniband/hw/amso1100/c2_rnic.c | 2 +-
drivers/infiniband/hw/cxgb3/iwch_provider.c | 3 +--
drivers/infiniband/hw/nes/nes_hw.c | 2 +-
include/rdma/ib_user_verbs.h | 4 ++++
include/rdma/ib_verbs.h | 9 +++++++--
6 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index 495c803..238680c 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -1492,6 +1492,10 @@ ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file,
next->wr.atomic.swap = user_wr->wr.atomic.swap;
next->wr.atomic.rkey = user_wr->wr.atomic.rkey;
break;
+ case IB_WR_SEND:
+ next->wr.invalidate.rkey =
+ user_wr->wr.invalidate.rkey;
+ break;
default:
break;
}
diff --git a/drivers/infiniband/hw/amso1100/c2_rnic.c b/drivers/infiniband/hw/amso1100/c2_rnic.c
index 7a62552..b1441ae 100644
--- a/drivers/infiniband/hw/amso1100/c2_rnic.c
+++ b/drivers/infiniband/hw/amso1100/c2_rnic.c
@@ -455,7 +455,7 @@ int __devinit c2_rnic_init(struct c2_dev *c2dev)
IB_DEVICE_CURR_QP_STATE_MOD |
IB_DEVICE_SYS_IMAGE_GUID |
IB_DEVICE_ZERO_STAG |
- IB_DEVICE_SEND_W_INV | IB_DEVICE_MEM_WINDOW);
+ IB_DEVICE_MEM_WINDOW);
/* Allocate the qptr_array */
c2dev->qptr_array = vmalloc(C2_MAX_CQS * sizeof(void *));
diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c
index 50e1f2a..ca72654 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_provider.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c
@@ -1109,8 +1109,7 @@ int iwch_register_device(struct iwch_dev *dev)
memcpy(&dev->ibdev.node_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
dev->ibdev.owner = THIS_MODULE;
dev->device_cap_flags =
- (IB_DEVICE_ZERO_STAG |
- IB_DEVICE_SEND_W_INV | IB_DEVICE_MEM_WINDOW);
+ (IB_DEVICE_ZERO_STAG | IB_DEVICE_MEM_WINDOW);
dev->ibdev.uverbs_cmd_mask =
(1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
diff --git a/drivers/infiniband/hw/nes/nes_hw.c b/drivers/infiniband/hw/nes/nes_hw.c
index 134189d..aa53aab 100644
--- a/drivers/infiniband/hw/nes/nes_hw.c
+++ b/drivers/infiniband/hw/nes/nes_hw.c
@@ -393,7 +393,7 @@ struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
nesadapter->base_pd = 1;
nesadapter->device_cap_flags =
- IB_DEVICE_ZERO_STAG | IB_DEVICE_SEND_W_INV | IB_DEVICE_MEM_WINDOW;
+ IB_DEVICE_ZERO_STAG | IB_DEVICE_MEM_WINDOW;
nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
[(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
diff --git a/include/rdma/ib_user_verbs.h b/include/rdma/ib_user_verbs.h
index 64a721f..7159cb8 100644
--- a/include/rdma/ib_user_verbs.h
+++ b/include/rdma/ib_user_verbs.h
@@ -553,6 +553,10 @@ struct ib_uverbs_send_wr {
__u32 remote_qkey;
__u32 reserved;
} ud;
+ struct {
+ __u32 rkey;
+ __u32 reserved;
+ } invalidate;
} wr;
};
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 40ff512..51fc1f7 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -94,7 +94,7 @@ enum ib_device_cap_flags {
IB_DEVICE_SRQ_RESIZE = (1<<13),
IB_DEVICE_N_NOTIFY_CQ = (1<<14),
IB_DEVICE_ZERO_STAG = (1<<15),
- IB_DEVICE_SEND_W_INV = (1<<16),
+ IB_DEVICE_RESERVED = (1<<16), /* old SEND_W_INV */
IB_DEVICE_MEM_WINDOW = (1<<17),
/*
* Devices should set IB_DEVICE_UD_IP_SUM if they support
@@ -104,6 +104,7 @@ enum ib_device_cap_flags {
* IPoIB driver may set NETIF_F_IP_CSUM for datagram mode.
*/
IB_DEVICE_UD_IP_CSUM = (1<<18),
+ IB_DEVICE_SEND_W_INV = (1<<21),
};
enum ib_atomic_cap {
@@ -625,7 +626,8 @@ enum ib_send_flags {
IB_SEND_SIGNALED = (1<<1),
IB_SEND_SOLICITED = (1<<2),
IB_SEND_INLINE = (1<<3),
- IB_SEND_IP_CSUM = (1<<4)
+ IB_SEND_IP_CSUM = (1<<4),
+ IB_SEND_INVALIDATE = (1<<6)
};
struct ib_sge {
@@ -660,6 +662,9 @@ struct ib_send_wr {
u16 pkey_index; /* valid for GSI only */
u8 port_num; /* valid for DR SMPs on switch only */
} ud;
+ struct {
+ u32 rkey;
+ } invalidate;
} wr;
};
--
1.5.4.3
More information about the general
mailing list