[openib-general] [PATCH] iwarp cm in kdapl
Matt Finlay
matt at ammasso.com
Wed Aug 31 09:18:12 PDT 2005
Tom,
Here is a patch against the IWarp branch that incorporates the IWarp CM calls into kDapl.
-Matt
Signed-off-by: Matt Finlay <matt at ammasso.com>
Index: ulp/kdapl/ib/dapl_openib_cm.h
===================================================================
--- ulp/kdapl/ib/dapl_openib_cm.h (revision 3186)
+++ ulp/kdapl/ib/dapl_openib_cm.h (working copy)
@@ -38,6 +38,8 @@
#include "ib_sa.h"
#include "ib_at.h"
+#include "iw_cm.h"
+
struct dapl_cm_ctx {
struct ib_at_ib_route dapl_rt;
struct ib_sa_path_rec dapl_path;
@@ -47,6 +49,7 @@
struct dapl_ep *ep;
struct dapl_sp *sp;
struct sockaddr *remote_ia_address;
+ void *iw_cr_id;
spinlock_t lock;
wait_queue_head_t wait;
int retries;
Index: ulp/kdapl/ib/dapl_openib_util.c
===================================================================
--- ulp/kdapl/ib/dapl_openib_util.c (revision 3186)
+++ ulp/kdapl/ib/dapl_openib_util.c (working copy)
@@ -91,9 +91,9 @@
{
enum ib_access_flags value = 0;
- /*
- * if (DAT_MEM_PRIV_LOCAL_READ_FLAG & priv) do nothing
- */
+ if (DAT_MEM_PRIV_LOCAL_READ_FLAG & priv)
+ value |= IB_ACCESS_LOCAL_READ;
+
if (DAT_MEM_PRIV_LOCAL_WRITE_FLAG & priv)
value |= IB_ACCESS_LOCAL_WRITE;
Index: ulp/kdapl/ib/dapl.h
===================================================================
--- ulp/kdapl/ib/dapl.h (revision 3186)
+++ ulp/kdapl/ib/dapl.h (working copy)
@@ -282,6 +282,7 @@
/* maintenance fields */
boolean_t listening; /* PSP is registered & active */
struct ib_cm_id *cm_srvc_handle; /* Used by CM */
+ struct iw_listen_ep *iw_ep_handle;
struct list_head cr_list; /* CR pending queue */
int cr_list_count; /* count of CRs on queue */
};
Index: ulp/kdapl/ib/dapl_openib_cm.c
===================================================================
--- ulp/kdapl/ib/dapl_openib_cm.c (revision 3186)
+++ ulp/kdapl/ib/dapl_openib_cm.c (working copy)
@@ -423,6 +423,137 @@
dapl_evd_connection_callback(cm_ctx, event, NULL, cm_ctx->ep);
}
+static void dapl_destroy_iw_cm_ctx(struct dapl_cm_ctx *cm_ctx)
+{
+ unsigned long flags;
+ int in_callback;
+
+ spin_lock_irqsave(&cm_ctx->lock, flags);
+ cm_ctx->destroy = 1;
+ in_callback = cm_ctx->in_callback;
+ spin_unlock_irqrestore(&cm_ctx->lock, flags);
+
+ if (!in_callback) {
+ dapl_dbg_log(DAPL_DBG_TYPE_CM,
+ " >>> dapl_destroy_iw_cm_ctx: cm_ctx %p CM ID %p\n", cm_ctx);
+ if (cm_ctx->ep)
+ cm_ctx->ep->cm_ctx = NULL;
+ kfree(cm_ctx);
+ }
+}
+static struct dapl_cm_ctx * dapl_get_iw_cm_ctx(struct iw_cm_event *event,
+ void *iw_ctx)
+{
+ struct dapl_cm_ctx *cm_ctx;
+
+ switch (event->event) {
+ case IW_EVENT_ACTIVE_CONNECT_RESULTS:
+ return (struct dapl_cm_ctx *)iw_ctx;
+ break;
+ case IW_EVENT_CONNECT_REQUEST: {
+ cm_ctx = kmalloc(sizeof *cm_ctx, GFP_KERNEL);
+ if (!cm_ctx) {
+ /* XXX reject? */
+ return NULL;
+ }
+
+ memset(cm_ctx, 0, sizeof *cm_ctx);
+ cm_ctx->sp = (struct dapl_sp *)iw_ctx;
+ spin_lock_init(&cm_ctx->lock);
+ init_waitqueue_head(&cm_ctx->wait);
+ return cm_ctx;
+ }
+ default:
+ return NULL;
+ break;
+ }
+
+}
+
+void dapl_iwarp_cm_cb_handler(struct iw_cm_event *event,
+ void *iw_ctx)
+{
+ struct dapl_cm_ctx *cm_ctx;
+ int ret;
+ unsigned long flags;
+ enum dat_event_number dat_event;
+
+ cm_ctx = dapl_get_iw_cm_ctx(event, iw_ctx);
+ if (!cm_ctx) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR, "dapl_iwarp_cm_cb_handler:"
+ " dapl_get_iw_cm_ctx failed\n");
+ return;
+ }
+
+ spin_lock_irqsave(&cm_ctx->lock, flags);
+ if (cm_ctx->destroy) {
+ spin_unlock_irqrestore(&cm_ctx->lock, flags);
+ return;
+ }
+ cm_ctx->in_callback = 1;
+ spin_unlock_irqrestore(&cm_ctx->lock, flags);
+
+ switch (event->event) {
+ case IW_EVENT_ACTIVE_CONNECT_RESULTS: {
+ struct iw_conn_results results;
+ u8 *private_data = NULL;
+
+ results = event->element.active_results;
+
+ switch(results.result) {
+ case IW_CONN_ACCEPT:
+ dat_event = DAT_CONNECTION_EVENT_ESTABLISHED;
+ private_data = results.private_data;
+ break;
+ case IW_CONN_RESET:
+ dat_event = DAT_CONNECTION_EVENT_NON_PEER_REJECTED;
+ break;
+ case IW_CONN_PEER_REJECT:
+ dat_event = DAT_CONNECTION_EVENT_PEER_REJECTED;
+ break;
+ case IW_CONN_TIMEDOUT:
+ dat_event = DAT_CONNECTION_EVENT_TIMED_OUT;
+ break;
+ case IW_CONN_NO_ROUTE_TO_HOST:
+ dat_event = DAT_CONNECTION_EVENT_UNREACHABLE;
+ break;
+ default:
+ dat_event = DAT_CONNECTION_EVENT_BROKEN;
+ break;
+ }
+
+ dapl_evd_connection_callback(cm_ctx, dat_event, private_data, cm_ctx->ep);
+ break;
+ }
+ case IW_EVENT_CONNECT_REQUEST: {
+ struct iw_conn_request request;
+
+ request = event->element.conn_request;
+ cm_ctx->iw_cr_id = request.cr_id;
+ dapl_cr_callback(cm_ctx, DAT_CONNECTION_REQUEST_EVENT,
+ request.private_data, cm_ctx->sp);
+
+ break;
+ }
+ default:
+ dapl_dbg_log(DAPL_DBG_TYPE_CM, "Unknown IWarp connection event: %d\n",
+ event->event);
+ break;
+ }
+
+ spin_lock_irqsave(&cm_ctx->lock, flags);
+ ret = cm_ctx->destroy;
+ cm_ctx->in_callback = cm_ctx->destroy;
+ spin_unlock_irqrestore(&cm_ctx->lock, flags);
+ if (ret) {
+ if (cm_ctx->ep)
+ cm_ctx->ep->cm_ctx = NULL;
+ kfree(cm_ctx);
+ }
+
+ return;
+}
+
/*
* dapl_ib_connect
*
@@ -452,6 +583,7 @@
{
struct dapl_ia *ia;
struct dapl_cm_ctx *cm_ctx;
+ struct ib_device *ib_dev = ep->common.owner_ia->provider->device;
int status;
if (ep->qp == NULL) {
@@ -470,11 +602,14 @@
spin_lock_init(&cm_ctx->lock);
init_waitqueue_head(&cm_ctx->wait);
cm_ctx->ep = ep;
- cm_ctx->cm_id = ib_create_cm_id(dapl_cm_active_cb_handler, cm_ctx);
- if (IS_ERR(cm_ctx->cm_id)) {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR, " CM ID creation failed\n");
- kfree(cm_ctx);
- return -EAGAIN;
+
+ if (ib_dev->node_type != IB_NODE_RNIC) {
+ cm_ctx->cm_id = ib_create_cm_id(dapl_cm_active_cb_handler, cm_ctx);
+ if (IS_ERR(cm_ctx->cm_id)) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR, " CM ID creation failed\n");
+ kfree(cm_ctx);
+ return -EAGAIN;
+ }
}
cm_ctx->ep->cm_ctx = cm_ctx;
@@ -501,20 +636,46 @@
cm_ctx->dapl_comp.context = cm_ctx;
cm_ctx->retries = 0;
cm_ctx->in_callback = 1;
- status =
- ib_at_route_by_ip(((struct sockaddr_in *)remote_ia_address)->
- sin_addr.s_addr, 0, 0, 0, &cm_ctx->dapl_rt,
- &cm_ctx->dapl_comp);
- if (status < 0) {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR, " ib_at_route_by_ip failed "
- "with status %d\n", status);
- kfree(cm_ctx);
- return -EAGAIN;
+
+ if (ib_dev->node_type == IB_NODE_RNIC) {
+
+ struct iw_conn_attr conn_attr;
+
+ cm_ctx->in_callback = 0;
+
+ /* Fill out the connection attributes */
+ memset(&conn_attr, 0, sizeof(conn_attr));
+ conn_attr.remote_addr = ((struct sockaddr_in *)remote_ia_address)->sin_addr;
+ conn_attr.remote_port = htons((u16)remote_conn_qual);
+
+ /* Issue the connect */
+ status = (*ib_dev->iwcm->connect_qp)(ep->qp,
+ &conn_attr,
+ dapl_iwarp_cm_cb_handler,
+ cm_ctx,
+ private_data,
+ private_data_size);
+
+ if (status < 0) {
+ kfree(cm_ctx);
+ return status;
+ }
+ } else {
+ status =
+ ib_at_route_by_ip(((struct sockaddr_in *)remote_ia_address)->
+ sin_addr.s_addr, 0, 0, 0, &cm_ctx->dapl_rt,
+ &cm_ctx->dapl_comp);
+ if (status < 0) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR, " ib_at_route_by_ip failed "
+ "with status %d\n", status);
+ kfree(cm_ctx);
+ return -EAGAIN;
+ }
+
+ if (status == 1)
+ dapl_rt_comp_handler(cm_ctx->dapl_comp.req_id, cm_ctx, 1);
}
- if (status == 1)
- dapl_rt_comp_handler(cm_ctx->dapl_comp.req_id, cm_ctx, 1);
-
return 0;
}
@@ -539,6 +700,7 @@
int dapl_ib_disconnect(struct dapl_ep *ep, enum dat_close_flags close_flags)
{
struct dapl_cm_ctx *cm_ctx = ep->cm_ctx;
+ struct ib_device *ib_dev;
int status;
dapl_dbg_log(DAPL_DBG_TYPE_CM,
@@ -548,13 +710,24 @@
if (cm_ctx == NULL)
return 0;
- if (close_flags == DAT_CLOSE_ABRUPT_FLAG)
- dapl_destroy_cm_id(cm_ctx);
- else {
- status = ib_send_cm_dreq(cm_ctx->cm_id, NULL, 0);
- if (status)
- printk(KERN_ERR "dapl_ib_disconnect: CM ID 0x%p "
- "status %d\n", ep->cm_ctx, status);
+ ib_dev = ep->common.owner_ia->provider->device;
+ if (ib_dev->node_type == IB_NODE_RNIC) {
+
+ status = (*ib_dev->iwcm->disconnect_qp)(ep->qp,
+ close_flags == DAT_CLOSE_ABRUPT_FLAG ? 1 : 0);
+ if (status < 0) {
+ return status;
+ }
+
+ } else {
+ if (close_flags == DAT_CLOSE_ABRUPT_FLAG)
+ dapl_destroy_cm_id(cm_ctx);
+ else {
+ status = ib_send_cm_dreq(cm_ctx->cm_id, NULL, 0);
+ if (status)
+ printk(KERN_ERR "dapl_ib_disconnect: CM ID 0x%p "
+ "status %d\n", ep->cm_ctx, status);
+ }
}
return 0;
@@ -669,20 +842,41 @@
int dapl_ib_setup_conn_listener(struct dapl_ia *ia, struct dapl_sp *sp)
{
int status;
+ struct ib_device *ib_dev = ia->provider->device;
- sp->cm_srvc_handle = ib_create_cm_id(dapl_cm_passive_cb_handler, sp);
- if (IS_ERR(sp->cm_srvc_handle)) {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR, " CM ID creation failed\n");
- return -EAGAIN;
- }
+ if (ib_dev->node_type == IB_NODE_RNIC) {
- status = ib_cm_listen(sp->cm_srvc_handle, cpu_to_be64(sp->conn_qual),
- 0);
- if (status) {
- ib_destroy_cm_id(sp->cm_srvc_handle);
- sp->cm_srvc_handle = NULL;
+ struct iw_listen_ep_attr listen_ep_attrs;
+ struct iw_listen_ep *ep_handle;
+
+ listen_ep_attrs.event_handler = dapl_iwarp_cm_cb_handler;
+ listen_ep_attrs.listen_context = sp;
+ listen_ep_attrs.addr.s_addr = INADDR_ANY;
+ listen_ep_attrs.port = htons((u16)sp->conn_qual);
+ listen_ep_attrs.backlog = ((struct dapl_evd *)sp->evd)->qlen;
- return status;
+ status = (*ib_dev->iwcm->create_listen_ep)(&listen_ep_attrs, &ep_handle);
+ if (status) {
+ return status;
+ }
+
+ sp->iw_ep_handle = ep_handle;
+
+ } else {
+ sp->cm_srvc_handle = ib_create_cm_id(dapl_cm_passive_cb_handler, sp);
+ if (IS_ERR(sp->cm_srvc_handle)) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR, " CM ID creation failed\n");
+ return -EAGAIN;
+ }
+
+ status = ib_cm_listen(sp->cm_srvc_handle, cpu_to_be64(sp->conn_qual),
+ 0);
+ if (status) {
+ ib_destroy_cm_id(sp->cm_srvc_handle);
+ sp->cm_srvc_handle = NULL;
+
+ return status;
+ }
}
return 0;
@@ -707,17 +901,32 @@
*/
int dapl_ib_remove_conn_listener(struct dapl_ia *ia, struct dapl_sp *sp)
{
+ struct ib_device *ib_dev = ia->provider->device;
+ int status;
+
dapl_dbg_log(DAPL_DBG_TYPE_CM,
" >>> dapl_ib_remove_conn_listener: SP %p conn %p\n",
sp, sp->cm_srvc_handle);
- /*
- * This will hang if called from CM thread context...
- * Move back to using WQ...
- */
- if (sp->cm_srvc_handle != NULL) {
- ib_destroy_cm_id(sp->cm_srvc_handle);
- sp->cm_srvc_handle = NULL;
+ if (ib_dev->node_type == IB_NODE_RNIC) {
+
+ if (sp->iw_ep_handle != NULL) {
+
+ status = (*ib_dev->iwcm->destroy_listen_ep)(sp->iw_ep_handle);
+ if (status) {
+ return status;
+ }
+ sp->iw_ep_handle = NULL;
+ }
+ } else {
+ /*
+ * This will hang if called from CM thread context...
+ * Move back to using WQ...
+ */
+ if (sp->cm_srvc_handle != NULL) {
+ ib_destroy_cm_id(sp->cm_srvc_handle);
+ sp->cm_srvc_handle = NULL;
+ }
}
return 0;
}
@@ -742,6 +951,7 @@
int dapl_ib_reject_connection(struct dapl_cm_ctx *cm_ctx)
{
int status;
+ struct ib_device *ib_dev;
if (cm_ctx == NULL) {
dapl_dbg_log(DAPL_DBG_TYPE_ERR,
@@ -749,14 +959,28 @@
return 0;
}
- status = ib_send_cm_rej(cm_ctx->cm_id, IB_CM_REJ_CONSUMER_DEFINED,
- NULL, 0, NULL, 0);
- if (status) {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR, " dapl_ib_reject_connection: "
- "ib_send_cm_rej failed: %d\n", status);
- return status;
+ ib_dev = cm_ctx->sp->common.owner_ia->provider->device;
+ if (ib_dev->node_type == IB_NODE_RNIC) {
+ status = (*ib_dev->iwcm->reject_cr)(ib_dev,
+ cm_ctx->iw_cr_id,
+ NULL,
+ 0);
+ if (status) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR, " dapl_ib_reject_connection: "
+ "reject_cr failed: %d\n", status);
+ return status;
+ }
+ dapl_destroy_iw_cm_ctx(cm_ctx);
+ } else {
+ status = ib_send_cm_rej(cm_ctx->cm_id, IB_CM_REJ_CONSUMER_DEFINED,
+ NULL, 0, NULL, 0);
+ if (status) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR, " dapl_ib_reject_connection: "
+ "ib_send_cm_rej failed: %d\n", status);
+ return status;
+ }
+ dapl_destroy_cm_id(cm_ctx);
}
- dapl_destroy_cm_id(cm_ctx);
return 0;
}
@@ -788,6 +1012,7 @@
int status;
struct ib_cm_rep_param passive_params;
struct dapl_cm_ctx *cm_ctx;
+ struct ib_device *ib_dev;
ia = ep->common.owner_ia;
cm_ctx = cr->cm_ctx;
@@ -818,28 +1043,42 @@
ep->cm_ctx = cm_ctx;
cm_ctx->ep = ep;
- memset(&passive_params, 0, sizeof passive_params);
- passive_params.private_data = priv_data;
- passive_params.private_data_len = priv_size;
- passive_params.qp_num = ep->qp->qp_num;
- passive_params.responder_resources = DAPL_IB_TARGET_MAX;
- passive_params.initiator_depth = DAPL_IB_INITIATOR_DEPTH;
- passive_params.rnr_retry_count = DAPL_IB_RNR_RETRY_COUNT;
+ ib_dev = ia->provider->device;
+ if (ib_dev->node_type == IB_NODE_RNIC) {
- /* Transition QP to RTR */
- status = dapl_modify_qp_state_to_rtr(cm_ctx->cm_id, ep->qp);
- if (status) {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR, " dapl_ib_accept_connection: "
- "could not modify QP state to RTR status %d\n",
- status);
- goto reject;
- }
+ status = (*ib_dev->iwcm->accept_cr)(ib_dev,
+ cm_ctx->iw_cr_id,
+ ep->qp,
+ (void *)priv_data,
+ priv_size);
+ if (status) {
+ dapl_destroy_iw_cm_ctx(cm_ctx);
+ return status;
+ }
+ } else {
+ memset(&passive_params, 0, sizeof passive_params);
+ passive_params.private_data = priv_data;
+ passive_params.private_data_len = priv_size;
+ passive_params.qp_num = ep->qp->qp_num;
+ passive_params.responder_resources = DAPL_IB_TARGET_MAX;
+ passive_params.initiator_depth = DAPL_IB_INITIATOR_DEPTH;
+ passive_params.rnr_retry_count = DAPL_IB_RNR_RETRY_COUNT;
- status = ib_send_cm_rep(cm_ctx->cm_id, &passive_params);
- if (status) {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR, " dapl_ib_accept_connection: "
- "ib_send_cm_rep failed: %d\n", status);
- goto reject;
+ /* Transition QP to RTR */
+ status = dapl_modify_qp_state_to_rtr(cm_ctx->cm_id, ep->qp);
+ if (status) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR, " dapl_ib_accept_connection: "
+ "could not modify QP state to RTR status %d\n",
+ status);
+ goto reject;
+ }
+
+ status = ib_send_cm_rep(cm_ctx->cm_id, &passive_params);
+ if (status) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR, " dapl_ib_accept_connection: "
+ "ib_send_cm_rep failed: %d\n", status);
+ goto reject;
+ }
}
return 0;
Index: include/ib_verbs.h
===================================================================
--- include/ib_verbs.h (revision 3275)
+++ include/ib_verbs.h (working copy)
@@ -617,10 +617,11 @@
enum ib_access_flags {
IB_ACCESS_LOCAL_WRITE = 1,
- IB_ACCESS_REMOTE_WRITE = (1<<1),
- IB_ACCESS_REMOTE_READ = (1<<2),
- IB_ACCESS_REMOTE_ATOMIC = (1<<3),
- IB_ACCESS_MW_BIND = (1<<4)
+ IB_ACCESS_LOCAL_READ = (1<<1),
+ IB_ACCESS_REMOTE_WRITE = (1<<2),
+ IB_ACCESS_REMOTE_READ = (1<<3),
+ IB_ACCESS_REMOTE_ATOMIC = (1<<4),
+ IB_ACCESS_MW_BIND = (1<<5)
};
struct ib_phys_buf {
Index: include/iw_cm.h
===================================================================
--- include/iw_cm.h (revision 3275)
+++ include/iw_cm.h (working copy)
@@ -119,6 +119,10 @@
int pdata_len
);
+ int (*disconnect_qp)(struct ib_qp *qp,
+ int abrupt
+ );
+
int (*accept_cr)(struct ib_device* ibdev,
void* cr_id,
struct ib_qp *qp,
More information about the general
mailing list