[ofw] [PATCH 1/11] librdmacm: support sync operation
Hefty, Sean
sean.hefty at intel.com
Mon Aug 16 16:13:45 PDT 2010
Support synchronous operation for address, route, and connections.
From: Sean Hefty <sean.hefty at intel.com>
---
trunk/ulp/librdmacm/include/rdma/rdma_cma.h | 2
trunk/ulp/librdmacm/src/cma.cpp | 124 ++++++++++++++++++++++-----
2 files changed, 102 insertions(+), 24 deletions(-)
diff --git a/trunk/ulp/librdmacm/include/rdma/rdma_cma.h b/trunk/ulp/librdmacm/include/rdma/rdma_cma.h
index 0c9d305..d57490e 100644
--- a/trunk/ulp/librdmacm/include/rdma/rdma_cma.h
+++ b/trunk/ulp/librdmacm/include/rdma/rdma_cma.h
@@ -132,6 +132,8 @@ struct rdma_cm_id
IWVConnectEndpoint *connect;
IWVDatagramEndpoint *datagram;
} ep;
+
+ struct rdma_cm_event *event;
};
struct rdma_conn_param
diff --git a/trunk/ulp/librdmacm/src/cma.cpp b/trunk/ulp/librdmacm/src/cma.cpp
index 1a96059..2698bde 100644
--- a/trunk/ulp/librdmacm/src/cma.cpp
+++ b/trunk/ulp/librdmacm/src/cma.cpp
@@ -33,6 +33,7 @@
#include <iphlpapi.h>
#include <rdma/rdma_cma.h>
+#include <rdma/rdma_verbs.h>
#include <infiniband/verbs.h>
#include <_errno.h>
#include <comp_channel.h>
@@ -68,6 +69,7 @@ struct cma_id_private
struct rdma_cm_id id;
enum cma_state state;
struct cma_device *cma_dev;
+ int sync;
int backlog;
int index;
volatile LONG refcnt;
@@ -261,9 +263,18 @@ int rdma_create_id(struct rdma_event_channel *channel,
RtlZeroMemory(id_priv, sizeof(struct cma_id_private));
id_priv->refcnt = 1;
id_priv->id.context = context;
- id_priv->id.channel = channel;
+
+ if (!channel) {
+ id_priv->id.channel = rdma_create_event_channel();
+ if (!id_priv->id.channel) {
+ goto err2;
+ }
+ id_priv->sync = 1;
+ } else {
+ id_priv->id.channel = channel;
+ }
id_priv->id.ps = ps;
- CompEntryInit(&channel->channel, &id_priv->id.comp_entry);
+ CompEntryInit(&id_priv->id.channel->channel, &id_priv->id.comp_entry);
if (ps == RDMA_PS_TCP) {
hr = windata.prov->CreateConnectEndpoint(&id_priv->id.ep.connect);
@@ -328,10 +339,16 @@ int rdma_destroy_id(struct rdma_cm_id *id)
id_priv->id.ep.datagram->Release();
}
+ if (id->event) {
+ rdma_ack_cm_event(id->event);
+ }
InterlockedDecrement(&id_priv->refcnt);
while (id_priv->refcnt) {
Sleep(0);
}
+ if (id_priv->sync) {
+ rdma_destroy_event_channel(id->channel);
+ }
delete id_priv;
ucma_release();
return 0;
@@ -471,6 +488,27 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
return ret;
}
+static int ucma_complete(struct cma_id_private *id_priv)
+{
+ int ret;
+
+ if (!id_priv->sync) {
+ return 0;
+ }
+
+ if (id_priv->id.event) {
+ rdma_ack_cm_event(id_priv->id.event);
+ id_priv->id.event = NULL;
+ }
+
+ ret = rdma_get_cm_event(id_priv->id.channel, &id_priv->id.event);
+ if (ret) {
+ return ret;
+ }
+
+ return id_priv->id.event->status;
+}
+
__declspec(dllexport)
int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
struct sockaddr *dst_addr, int timeout_ms)
@@ -514,7 +552,7 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
id_priv->refcnt++;
CompEntryPost(&id->comp_entry);
- return 0;
+ return ucma_complete(id_priv);
}
__declspec(dllexport)
@@ -542,7 +580,7 @@ int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
id_priv->refcnt++;
CompEntryPost(&id->comp_entry);
- return 0;
+ return ucma_complete(id_priv);
}
static int ucma_modify_qp_init(struct cma_id_private *id_priv, struct ibv_qp *qp)
@@ -678,7 +716,7 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
return ibvw_wv_errno(hr);
}
- return 0;
+ return ucma_complete(id_priv);
}
static int ucma_get_request(struct cma_id_private *listen, int index)
@@ -805,7 +843,7 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
return ibvw_wv_errno(hr);
}
- return 0;
+ return ucma_complete(id_priv);
}
__declspec(dllexport)
@@ -847,7 +885,7 @@ int rdma_disconnect(struct rdma_cm_id *id)
return ibvw_wv_errno(hr);
}
- return 0;
+ return ucma_complete(id_priv);
}
__declspec(dllexport)
@@ -876,21 +914,33 @@ static int ucma_process_conn_req(struct cma_event *event)
ucma_get_request(listen, id_priv->index);
- if (!event->event.status) {
- event->event.status = ucma_query_connect(&id_priv->id,
- &event->event.param.conn);
+ if (event->event.status) {
+ goto err;
}
- if (!event->event.status) {
- event->event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
- id_priv->state = cma_passive_connect;
- event->event.listen_id = &listen->id;
- } else {
- InterlockedDecrement(&listen->refcnt);
- InterlockedDecrement(&id_priv->refcnt);
- rdma_destroy_id(&id_priv->id);
+ if (listen->sync) {
+ event->event.status = rdma_migrate_id(&id_priv->id, NULL);
+ if (event->event.status) {
+ goto err;
+ }
+ }
+
+ event->event.status = ucma_query_connect(&id_priv->id,
+ &event->event.param.conn);
+ if (event->event.status) {
+ goto err;
}
+ event->event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
+ id_priv->state = cma_passive_connect;
+ event->event.listen_id = &listen->id;
+
+ return 0;
+
+err:
+ InterlockedDecrement(&listen->refcnt);
+ InterlockedDecrement(&id_priv->refcnt);
+ rdma_destroy_id(&id_priv->id);
return event->event.status;
}
@@ -1043,15 +1093,13 @@ __declspec(dllexport)
int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
void *context)
{
- _set_errno(ENOSYS);
- return -1;
+ return rdma_seterrno(ENOSYS);
}
__declspec(dllexport)
int rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
{
- _set_errno(ENOSYS);
- return -1;
+ return rdma_seterrno(ENOSYS);
}
__declspec(dllexport)
@@ -1099,14 +1147,42 @@ __declspec(dllexport)
int rdma_set_option(struct rdma_cm_id *id, int level, int optname,
void *optval, size_t optlen)
{
- _set_errno(ENOSYS);
- return -1;
+ return rdma_seterrno(ENOSYS);
}
__declspec(dllexport)
int rdma_migrate_id(struct rdma_cm_id *id, struct rdma_event_channel *channel)
{
+ struct cma_id_private *id_priv;
+ int sync;
+
+ id_priv = CONTAINING_RECORD(id, struct cma_id_private, id);
+ if (id_priv->sync && !channel) {
+ return rdma_seterrno(EINVAL);
+ }
+
+ if (id->comp_entry.Busy) {
+ return rdma_seterrno(EBUSY);
+ }
+
+ if ((sync = (channel == NULL))) {
+ channel = rdma_create_event_channel();
+ if (!channel) {
+ return rdma_seterrno(ENOMEM);
+ }
+ }
+
+ if (id_priv->sync) {
+ if (id->event) {
+ rdma_ack_cm_event(id->event);
+ id->event = NULL;
+ }
+ rdma_destroy_event_channel(id->channel);
+ }
+
+ id_priv->sync = sync;
id->channel = channel;
+ id->comp_entry.Channel = &channel->channel;
return 0;
}
More information about the ofw
mailing list