[openib-general] [PATCH] [uCM] user specified context to uCM
Sean Hefty
sean.hefty at intel.com
Sat Aug 20 11:22:14 PDT 2005
Try #3 - I'm not seeing this message make it through...
The following patch:
* Adds user specified context to all uCM events. Users will not retrieve
any events associated with the context after destroying the corresponding
cm_id.
* Provides the ib_cm_init_qp_attr() call to userspace clients of the CM.
This call may be used to set QP attributes properly before modifying the QP.
* Fixes some error handling synchronization and cleanup issues.
* Performs some minor code cleanup.
* Replaces the ucm_simple test program with a userspace version of cmpost.
The userspace version of cmpost uses the uAT interface to retrieve path
records based on a remote host name, establishes a connection over a QP,
and performs some simple message passing between the nodes.
This patch bumps the ABI, and will require synchronization with uDAPL before
committing.
Signed-off-by: Sean Hefty <sean.hefty at intel.com>
Index: userspace/libibcm/include/infiniband/cm_abi.h
===================================================================
--- userspace/libibcm/include/infiniband/cm_abi.h (revision 3124)
+++ userspace/libibcm/include/infiniband/cm_abi.h (working copy)
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2005 Topspin Communications. All rights reserved.
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -41,7 +42,7 @@
* drivers/infiniband/include/ib_user_cm.h
*/
-#define IB_USER_CM_ABI_VERSION 1
+#define IB_USER_CM_ABI_VERSION 2
enum {
IB_USER_CM_CMD_CREATE_ID,
@@ -64,6 +65,7 @@ enum {
IB_USER_CM_CMD_SEND_SIDR_REP,
IB_USER_CM_CMD_EVENT,
+ IB_USER_CM_CMD_INIT_QP_ATTR,
};
/*
* command ABI structures.
@@ -75,6 +77,7 @@ struct cm_abi_cmd_hdr {
};
struct cm_abi_create_id {
+ __u64 uid;
__u64 response;
};
@@ -83,9 +86,14 @@ struct cm_abi_create_id_resp {
};
struct cm_abi_destroy_id {
+ __u64 response;
__u32 id;
};
+struct cm_abi_destroy_id_resp {
+ __u32 events_reported;
+};
+
struct cm_abi_attr_id {
__u64 response;
__u32 id;
@@ -98,6 +106,64 @@ struct cm_abi_attr_id_resp {
__u32 remote_id;
};
+struct cm_abi_init_qp_attr {
+ __u64 response;
+ __u32 id;
+ __u32 qp_state;
+};
+
+struct cm_abi_ah_attr {
+ __u8 grh_dgid[16];
+ __u32 grh_flow_label;
+ __u16 dlid;
+ __u16 reserved;
+ __u8 grh_sgid_index;
+ __u8 grh_hop_limit;
+ __u8 grh_traffic_class;
+ __u8 sl;
+ __u8 src_path_bits;
+ __u8 static_rate;
+ __u8 is_global;
+ __u8 port_num;
+};
+
+struct cm_abi_init_qp_attr_resp {
+ __u32 qp_attr_mask;
+ __u32 qp_state;
+ __u32 cur_qp_state;
+ __u32 path_mtu;
+ __u32 path_mig_state;
+ __u32 qkey;
+ __u32 rq_psn;
+ __u32 sq_psn;
+ __u32 dest_qp_num;
+ __u32 qp_access_flags;
+
+ struct cm_abi_ah_attr ah_attr;
+ struct cm_abi_ah_attr alt_ah_attr;
+
+ /* ibv_qp_cap */
+ __u32 max_send_wr;
+ __u32 max_recv_wr;
+ __u32 max_send_sge;
+ __u32 max_recv_sge;
+ __u32 max_inline_data;
+
+ __u16 pkey_index;
+ __u16 alt_pkey_index;
+ __u8 en_sqd_async_notify;
+ __u8 sq_draining;
+ __u8 max_rd_atomic;
+ __u8 max_dest_rd_atomic;
+ __u8 min_rnr_timer;
+ __u8 port_num;
+ __u8 timeout;
+ __u8 retry_cnt;
+ __u8 rnr_retry;
+ __u8 alt_port_num;
+ __u8 alt_timeout;
+};
+
struct cm_abi_listen {
__u64 service_id;
__u64 service_mask;
@@ -161,6 +227,7 @@ struct cm_abi_req {
};
struct cm_abi_rep {
+ __u64 uid;
__u64 data;
__u32 id;
__u32 qpn;
@@ -236,7 +303,6 @@ struct cm_abi_event_get {
};
struct cm_abi_req_event_resp {
- __u32 listen_id;
/* device */
/* port */
struct cm_abi_path_rec primary_path;
@@ -291,7 +357,6 @@ struct cm_abi_apr_event_resp {
};
struct cm_abi_sidr_req_event_resp {
- __u32 listen_id;
/* device */
/* port */
__u16 pkey;
@@ -311,6 +376,7 @@ struct cm_abi_sidr_rep_event_resp {
#define CM_ABI_PRES_ALTERNATE 0x08
struct cm_abi_event_resp {
+ __u64 uid;
__u32 id;
__u32 event;
__u32 present;
Index: userspace/libibcm/include/infiniband/cm.h
===================================================================
--- userspace/libibcm/include/infiniband/cm.h (revision 3124)
+++ userspace/libibcm/include/infiniband/cm.h (working copy)
@@ -77,8 +77,13 @@ enum ib_cm_data_size {
IB_CM_SIDR_REP_INFO_LENGTH = 72
};
+struct ib_cm_id {
+ void *context;
+ uint32_t handle;
+};
+
struct ib_cm_req_event_param {
- uint32_t listen_id;
+ struct ib_cm_id *listen_id;
struct ib_sa_path_rec *primary_path;
struct ib_sa_path_rec *alternate_path;
@@ -187,7 +192,7 @@ struct ib_cm_apr_event_param {
};
struct ib_cm_sidr_req_event_param {
- uint32_t listen_id;
+ struct ib_cm_id *listen_id;
struct ib_device *device;
uint8_t port;
uint16_t pkey;
@@ -212,7 +217,7 @@ struct ib_cm_sidr_rep_event_param {
};
struct ib_cm_event {
- uint32_t cm_id;
+ struct ib_cm_id *cm_id;
enum ib_cm_event_type event;
union {
struct ib_cm_req_event_param req_rcvd;
@@ -287,13 +292,13 @@ int ib_cm_get_fd(void);
* Communication identifiers are used to track connection states, service
* ID resolution requests, and listen requests.
*/
-int ib_cm_create_id(uint32_t *cm_id);
+int ib_cm_create_id(struct ib_cm_id **cm_id, void *context);
/**
* ib_cm_destroy_id - Destroy a connection identifier.
* @cm_id: Connection identifier to destroy.
*/
-int ib_cm_destroy_id(uint32_t cm_id);
+int ib_cm_destroy_id(struct ib_cm_id *cm_id);
struct ib_cm_attr_param {
uint64_t service_id;
@@ -309,7 +314,7 @@ struct ib_cm_attr_param {
*
* Not all parameters are valid during all connection states.
*/
-int ib_cm_attr_id(uint32_t cm_id,
+int ib_cm_attr_id(struct ib_cm_id *cm_id,
struct ib_cm_attr_param *param);
/**
@@ -323,7 +328,7 @@ int ib_cm_attr_id(uint32_t cm_id,
* range of service IDs. If set to 0, the service ID is matched
* exactly.
*/
-int ib_cm_listen(uint32_t cm_id,
+int ib_cm_listen(struct ib_cm_id *cm_id,
uint64_t service_id,
uint64_t service_mask);
@@ -355,7 +360,7 @@ struct ib_cm_req_param {
* @param: Connection request information needed to establish the
* connection.
*/
-int ib_cm_send_req(uint32_t cm_id,
+int ib_cm_send_req(struct ib_cm_id *cm_id,
struct ib_cm_req_param *param);
struct ib_cm_rep_param {
@@ -380,7 +385,7 @@ struct ib_cm_rep_param {
* @param: Connection reply information needed to establish the
* connection.
*/
-int ib_cm_send_rep(uint32_t cm_id,
+int ib_cm_send_rep(struct ib_cm_id *cm_id,
struct ib_cm_rep_param *param);
/**
@@ -391,7 +396,7 @@ int ib_cm_send_rep(uint32_t cm_id,
* ready to use message.
* @private_data_len: Size of the private data buffer, in bytes.
*/
-int ib_cm_send_rtu(uint32_t cm_id,
+int ib_cm_send_rtu(struct ib_cm_id *cm_id,
void *private_data,
uint8_t private_data_len);
@@ -404,7 +409,7 @@ int ib_cm_send_rtu(uint32_t cm_id,
* disconnection request message.
* @private_data_len: Size of the private data buffer, in bytes.
*/
-int ib_cm_send_dreq(uint32_t cm_id,
+int ib_cm_send_dreq(struct ib_cm_id *cm_id,
void *private_data,
uint8_t private_data_len);
@@ -416,7 +421,7 @@ int ib_cm_send_dreq(uint32_t cm_id,
* disconnection reply message.
* @private_data_len: Size of the private data buffer, in bytes.
*/
-int ib_cm_send_drep(uint32_t cm_id,
+int ib_cm_send_drep(struct ib_cm_id *cm_id,
void *private_data,
uint8_t private_data_len);
@@ -427,7 +432,7 @@ int ib_cm_send_drep(uint32_t cm_id,
* This routine should be invoked by users who receive messages on a
* connected QP before an RTU has been received.
*/
-int ib_cm_establish(uint32_t cm_id);
+int ib_cm_establish(struct ib_cm_id *cm_id);
/**
* ib_cm_send_rej - Sends a connection rejection message to the
@@ -441,7 +446,7 @@ int ib_cm_establish(uint32_t cm_id);
* rejection message.
* @private_data_len: Size of the private data buffer, in bytes.
*/
-int ib_cm_send_rej(uint32_t cm_id,
+int ib_cm_send_rej(struct ib_cm_id *cm_id,
enum ib_cm_rej_reason reason,
void *ari,
uint8_t ari_length,
@@ -458,7 +463,7 @@ int ib_cm_send_rej(uint32_t cm_id,
* message receipt acknowledgement.
* @private_data_len: Size of the private data buffer, in bytes.
*/
-int ib_cm_send_mra(uint32_t cm_id,
+int ib_cm_send_mra(struct ib_cm_id *cm_id,
uint8_t service_timeout,
void *private_data,
uint8_t private_data_len);
@@ -473,12 +478,32 @@ int ib_cm_send_mra(uint32_t cm_id,
* load alternate path message.
* @private_data_len: Size of the private data buffer, in bytes.
*/
-int ib_cm_send_lap(uint32_t cm_id,
+int ib_cm_send_lap(struct ib_cm_id *cm_id,
struct ib_sa_path_rec *alternate_path,
void *private_data,
uint8_t private_data_len);
/**
+ * ib_cm_init_qp_attr - Initializes the QP attributes for use in transitioning
+ * to a specified QP state.
+ * @cm_id: Communication identifier associated with the QP attributes to
+ * initialize.
+ * @qp_attr: On input, specifies the desired QP state. On output, the
+ * mandatory and desired optional attributes will be set in order to
+ * modify the QP to the specified state.
+ * @qp_attr_mask: The QP attribute mask that may be used to transition the
+ * QP to the specified state.
+ *
+ * Users must set the @qp_attr->qp_state to the desired QP state. This call
+ * will set all required attributes for the given transition, along with
+ * known optional attributes. Users may override the attributes returned from
+ * this call before calling ib_modify_qp.
+ */
+int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
+ struct ibv_qp_attr *qp_attr,
+ int *qp_attr_mask);
+
+/**
* ib_cm_send_apr - Sends an alternate path response message in response to
* a load alternate path request.
* @cm_id: Connection identifier associated with the alternate path response.
@@ -490,7 +515,7 @@ int ib_cm_send_lap(uint32_t cm_id,
* alternate path response message.
* @private_data_len: Size of the private data buffer, in bytes.
*/
-int ib_cm_send_apr(uint32_t cm_id,
+int ib_cm_send_apr(struct ib_cm_id *cm_id,
enum ib_cm_apr_status status,
void *info,
uint8_t info_length,
@@ -514,7 +539,7 @@ struct ib_cm_sidr_req_param {
* service ID resolution request.
* @param: Service ID resolution request information.
*/
-int ib_cm_send_sidr_req(uint32_t cm_id,
+int ib_cm_send_sidr_req(struct ib_cm_id *cm_id,
struct ib_cm_sidr_req_param *param);
struct ib_cm_sidr_rep_param {
@@ -534,7 +559,7 @@ struct ib_cm_sidr_rep_param {
* resolution request.
* @param: Service ID resolution reply information.
*/
-int ib_cm_send_sidr_rep(uint32_t cm_id,
+int ib_cm_send_sidr_rep(struct ib_cm_id *cm_id,
struct ib_cm_sidr_rep_param *param);
#endif /* CM_H */
Index: userspace/libibcm/AUTHORS
===================================================================
--- userspace/libibcm/AUTHORS (revision 3124)
+++ userspace/libibcm/AUTHORS (working copy)
@@ -1 +1,2 @@
+Sean Hefty <sean.hefty at intel.com>
Libor Michalek <libor at topspin.com>
Index: userspace/libibcm/src/cm.c
===================================================================
--- userspace/libibcm/src/cm.c (revision 3124)
+++ userspace/libibcm/src/cm.c (working copy)
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2005 Topspin Communications. All rights reserved.
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -45,6 +46,7 @@
#include <stdint.h>
#include <poll.h>
#include <unistd.h>
+#include <pthread.h>
#include <infiniband/cm.h>
#include <infiniband/cm_abi.h>
@@ -69,7 +71,7 @@ do {
resp = alloca(sizeof(*resp)); \
if (!resp) \
return -ENOMEM; \
- cmd->response = (unsigned long)resp;\
+ cmd->response = (uintptr_t)resp;\
} while (0)
#define CM_CREATE_MSG_CMD(msg, cmd, type, size) \
@@ -88,8 +90,18 @@ do {
memset(cmd, 0, sizeof(*cmd)); \
} while (0)
+struct cm_id_private {
+ struct ib_cm_id id;
+ int events_completed;
+ pthread_cond_t cond;
+ pthread_mutex_t mut;
+};
+
static int fd;
+#define container_of(ptr, type, field) \
+ ((type *) ((void *)ptr - offsetof(type, field)))
+
static void __attribute__((constructor)) ib_cm_init(void)
{
fd = open(IB_UCM_DEV_PATH, O_RDWR);
@@ -127,46 +139,89 @@ static void cm_param_path_get(struct cm_
abi->preference = sa->preference;
}
-int ib_cm_create_id(uint32_t *cm_id)
+static void ib_cm_free_id(struct cm_id_private *cm_id_priv)
+{
+ pthread_cond_destroy(&cm_id_priv->cond);
+ pthread_mutex_destroy(&cm_id_priv->mut);
+ free(cm_id_priv);
+}
+
+static struct cm_id_private *ib_cm_alloc_id(void *context)
+{
+ struct cm_id_private *cm_id_priv;
+
+ cm_id_priv = malloc(sizeof *cm_id_priv);
+ if (!cm_id_priv)
+ return NULL;
+
+ memset(cm_id_priv, 0, sizeof *cm_id_priv);
+ cm_id_priv->id.context = context;
+ pthread_mutex_init(&cm_id_priv->mut, NULL);
+ if (pthread_cond_init(&cm_id_priv->cond, NULL))
+ goto err;
+
+ return cm_id_priv;
+
+err: ib_cm_free_id(cm_id_priv);
+ return NULL;
+}
+
+int ib_cm_create_id(struct ib_cm_id **cm_id, void *context)
{
struct cm_abi_create_id_resp *resp;
struct cm_abi_create_id *cmd;
+ struct cm_id_private *cm_id_priv;
void *msg;
int result;
int size;
- if (!cm_id)
- return -EINVAL;
+ cm_id_priv = ib_cm_alloc_id(context);
+ if (!cm_id_priv)
+ return -ENOMEM;
- CM_CREATE_MSG_CMD_RESP(msg, cmd, resp, IB_USER_CM_CMD_CREATE_ID, size);
+ CM_CREATE_MSG_CMD_RESP(msg, cmd, resp, IB_USER_CM_CMD_CREATE_ID, size);
+ cmd->uid = (uintptr_t) cm_id_priv;
result = write(fd, msg, size);
if (result != size)
- return (result > 0) ? -ENODATA : result;
+ goto err;
- *cm_id = resp->id;
+ cm_id_priv->id.handle = resp->id;
+ *cm_id = &cm_id_priv->id;
return 0;
+
+err: ib_cm_free_id(cm_id_priv);
+ return result;
}
-int ib_cm_destroy_id(uint32_t cm_id)
+int ib_cm_destroy_id(struct ib_cm_id *cm_id)
{
+ struct cm_abi_destroy_id_resp *resp;
struct cm_abi_destroy_id *cmd;
+ struct cm_id_private *cm_id_priv;
void *msg;
int result;
int size;
- CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_DESTROY_ID, size);
-
- cmd->id = cm_id;
+ CM_CREATE_MSG_CMD_RESP(msg, cmd, resp, IB_USER_CM_CMD_DESTROY_ID, size);
+ cmd->id = cm_id->handle;
result = write(fd, msg, size);
if (result != size)
return (result > 0) ? -ENODATA : result;
+ cm_id_priv = container_of(cm_id, struct cm_id_private, id);
+
+ pthread_mutex_lock(&cm_id_priv->mut);
+ while (cm_id_priv->events_completed < resp->events_reported)
+ pthread_cond_wait(&cm_id_priv->cond, &cm_id_priv->mut);
+ pthread_mutex_unlock(&cm_id_priv->mut);
+
+ ib_cm_free_id(cm_id_priv);
return 0;
}
-int ib_cm_attr_id(uint32_t cm_id, struct ib_cm_attr_param *param)
+int ib_cm_attr_id(struct ib_cm_id *cm_id, struct ib_cm_attr_param *param)
{
struct cm_abi_attr_id_resp *resp;
struct cm_abi_attr_id *cmd;
@@ -177,9 +232,8 @@ int ib_cm_attr_id(uint32_t cm_id, struct
if (!param)
return -EINVAL;
- CM_CREATE_MSG_CMD_RESP(msg, cmd, resp, IB_USER_CM_CMD_ATTR_ID, size);
-
- cmd->id = cm_id;
+ CM_CREATE_MSG_CMD_RESP(msg, cmd, resp, IB_USER_CM_CMD_ATTR_ID, size);
+ cmd->id = cm_id->handle;
result = write(fd, msg, size);
if (result != size)
@@ -189,11 +243,91 @@ int ib_cm_attr_id(uint32_t cm_id, struct
param->service_mask = resp->service_mask;
param->local_id = resp->local_id;
param->remote_id = resp->remote_id;
+ return 0;
+}
+
+static void ib_cm_copy_ah_attr(struct ibv_ah_attr *dest_attr,
+ struct cm_abi_ah_attr *src_attr)
+{
+ memcpy(dest_attr->grh.dgid.raw, src_attr->grh_dgid,
+ sizeof dest_attr->grh.dgid);
+ dest_attr->grh.flow_label = src_attr->grh_flow_label;
+ dest_attr->grh.sgid_index = src_attr->grh_sgid_index;
+ dest_attr->grh.hop_limit = src_attr->grh_hop_limit;
+ dest_attr->grh.traffic_class = src_attr->grh_traffic_class;
+
+ dest_attr->dlid = src_attr->dlid;
+ dest_attr->sl = src_attr->sl;
+ dest_attr->src_path_bits = src_attr->src_path_bits;
+ dest_attr->static_rate = src_attr->static_rate;
+ dest_attr->is_global = src_attr->is_global;
+ dest_attr->port_num = src_attr->port_num;
+}
+
+static void ib_cm_copy_qp_attr(struct ibv_qp_attr *dest_attr,
+ struct cm_abi_init_qp_attr_resp *src_attr)
+{
+ dest_attr->cur_qp_state = src_attr->cur_qp_state;
+ dest_attr->path_mtu = src_attr->path_mtu;
+ dest_attr->path_mig_state = src_attr->path_mig_state;
+ dest_attr->qkey = src_attr->qkey;
+ dest_attr->rq_psn = src_attr->rq_psn;
+ dest_attr->sq_psn = src_attr->sq_psn;
+ dest_attr->dest_qp_num = src_attr->dest_qp_num;
+ dest_attr->qp_access_flags = src_attr->qp_access_flags;
+
+ dest_attr->cap.max_send_wr = src_attr->max_send_wr;
+ dest_attr->cap.max_recv_wr = src_attr->max_recv_wr;
+ dest_attr->cap.max_send_sge = src_attr->max_send_sge;
+ dest_attr->cap.max_recv_sge = src_attr->max_recv_sge;
+ dest_attr->cap.max_inline_data = src_attr->max_inline_data;
+
+ ib_cm_copy_ah_attr(&dest_attr->ah_attr, &src_attr->ah_attr);
+ ib_cm_copy_ah_attr(&dest_attr->alt_ah_attr, &src_attr->alt_ah_attr);
+
+ dest_attr->pkey_index = src_attr->pkey_index;
+ dest_attr->alt_pkey_index = src_attr->alt_pkey_index;
+ dest_attr->en_sqd_async_notify = src_attr->en_sqd_async_notify;
+ dest_attr->sq_draining = src_attr->sq_draining;
+ dest_attr->max_rd_atomic = src_attr->max_rd_atomic;
+ dest_attr->max_dest_rd_atomic = src_attr->max_dest_rd_atomic;
+ dest_attr->min_rnr_timer = src_attr->min_rnr_timer;
+ dest_attr->port_num = src_attr->port_num;
+ dest_attr->timeout = src_attr->timeout;
+ dest_attr->retry_cnt = src_attr->retry_cnt;
+ dest_attr->rnr_retry = src_attr->rnr_retry;
+ dest_attr->alt_port_num = src_attr->alt_port_num;
+ dest_attr->alt_timeout = src_attr->alt_timeout;
+}
+
+int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
+ struct ibv_qp_attr *qp_attr,
+ int *qp_attr_mask)
+{
+ struct cm_abi_init_qp_attr_resp *resp;
+ struct cm_abi_init_qp_attr *cmd;
+ void *msg;
+ int result;
+ int size;
+
+ if (!qp_attr || !qp_attr_mask)
+ return -EINVAL;
+
+ CM_CREATE_MSG_CMD_RESP(msg, cmd, resp, IB_USER_CM_CMD_INIT_QP_ATTR, size);
+ cmd->id = cm_id->handle;
+ cmd->qp_state = qp_attr->qp_state;
+
+ result = write(fd, msg, size);
+ if (result != size)
+ return (result > 0) ? -ENODATA : result;
+
+ *qp_attr_mask = resp->qp_attr_mask;
+ ib_cm_copy_qp_attr(qp_attr, resp);
return 0;
}
-int ib_cm_listen(uint32_t cm_id,
+int ib_cm_listen(struct ib_cm_id *cm_id,
uint64_t service_id,
uint64_t service_mask)
{
@@ -203,8 +337,7 @@ int ib_cm_listen(uint32_t cm_id,
int size;
CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_LISTEN, size);
-
- cmd->id = cm_id;
+ cmd->id = cm_id->handle;
cmd->service_id = service_id;
cmd->service_mask = service_mask;
@@ -215,7 +348,7 @@ int ib_cm_listen(uint32_t cm_id,
return 0;
}
-int ib_cm_send_req(uint32_t cm_id, struct ib_cm_req_param *param)
+int ib_cm_send_req(struct ib_cm_id *cm_id, struct ib_cm_req_param *param)
{
struct cm_abi_path_rec *p_path;
struct cm_abi_path_rec *a_path;
@@ -228,13 +361,11 @@ int ib_cm_send_req(uint32_t cm_id, struc
return -EINVAL;
CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_SEND_REQ, size);
-
- cmd->id = cm_id;
- cmd->qpn = param->qp_num;
- cmd->qp_type = param->qp_type;
- cmd->psn = param->starting_psn;
- cmd->sid = param->service_id;
-
+ cmd->id = cm_id->handle;
+ cmd->qpn = param->qp_num;
+ cmd->qp_type = param->qp_type;
+ cmd->psn = param->starting_psn;
+ cmd->sid = param->service_id;
cmd->peer_to_peer = param->peer_to_peer;
cmd->responder_resources = param->responder_resources;
cmd->initiator_depth = param->initiator_depth;
@@ -247,28 +378,25 @@ int ib_cm_send_req(uint32_t cm_id, struc
cmd->srq = param->srq;
if (param->primary_path) {
-
p_path = alloca(sizeof(*p_path));
if (!p_path)
return -ENOMEM;
cm_param_path_get(p_path, param->primary_path);
- cmd->primary_path = (unsigned long)p_path;
+ cmd->primary_path = (uintptr_t) p_path;
}
if (param->alternate_path) {
-
a_path = alloca(sizeof(*a_path));
if (!a_path)
return -ENOMEM;
cm_param_path_get(a_path, param->alternate_path);
- cmd->alternate_path = (unsigned long)a_path;
+ cmd->alternate_path = (uintptr_t) a_path;
}
if (param->private_data && param->private_data_len) {
-
- cmd->data = (unsigned long)param->private_data;
+ cmd->data = (uintptr_t) param->private_data;
cmd->len = param->private_data_len;
}
@@ -279,7 +407,7 @@ int ib_cm_send_req(uint32_t cm_id, struc
return 0;
}
-int ib_cm_send_rep(uint32_t cm_id, struct ib_cm_rep_param *param)
+int ib_cm_send_rep(struct ib_cm_id *cm_id, struct ib_cm_rep_param *param)
{
struct cm_abi_rep *cmd;
void *msg;
@@ -290,11 +418,10 @@ int ib_cm_send_rep(uint32_t cm_id, struc
return -EINVAL;
CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_SEND_REP, size);
-
- cmd->id = cm_id;
- cmd->qpn = param->qp_num;
- cmd->psn = param->starting_psn;
-
+ cmd->uid = (uintptr_t) container_of(cm_id, struct cm_id_private, id);
+ cmd->id = cm_id->handle;
+ cmd->qpn = param->qp_num;
+ cmd->psn = param->starting_psn;
cmd->responder_resources = param->responder_resources;
cmd->initiator_depth = param->initiator_depth;
cmd->target_ack_delay = param->target_ack_delay;
@@ -304,8 +431,7 @@ int ib_cm_send_rep(uint32_t cm_id, struc
cmd->srq = param->srq;
if (param->private_data && param->private_data_len) {
-
- cmd->data = (unsigned long)param->private_data;
+ cmd->data = (uintptr_t) param->private_data;
cmd->len = param->private_data_len;
}
@@ -316,7 +442,7 @@ int ib_cm_send_rep(uint32_t cm_id, struc
return 0;
}
-static inline int cm_send_private_data(uint32_t cm_id,
+static inline int cm_send_private_data(struct ib_cm_id *cm_id,
uint32_t type,
void *private_data,
uint8_t private_data_len)
@@ -327,12 +453,10 @@ static inline int cm_send_private_data(u
int size;
CM_CREATE_MSG_CMD(msg, cmd, type, size);
-
- cmd->id = cm_id;
+ cmd->id = cm_id->handle;
if (private_data && private_data_len) {
-
- cmd->data = (unsigned long)private_data;
+ cmd->data = (uintptr_t) private_data;
cmd->len = private_data_len;
}
@@ -343,7 +467,7 @@ static inline int cm_send_private_data(u
return 0;
}
-int ib_cm_send_rtu(uint32_t cm_id,
+int ib_cm_send_rtu(struct ib_cm_id *cm_id,
void *private_data,
uint8_t private_data_len)
{
@@ -351,7 +475,7 @@ int ib_cm_send_rtu(uint32_t cm_id,
private_data, private_data_len);
}
-int ib_cm_send_dreq(uint32_t cm_id,
+int ib_cm_send_dreq(struct ib_cm_id *cm_id,
void *private_data,
uint8_t private_data_len)
{
@@ -359,7 +483,7 @@ int ib_cm_send_dreq(uint32_t cm_id,
private_data, private_data_len);
}
-int ib_cm_send_drep(uint32_t cm_id,
+int ib_cm_send_drep(struct ib_cm_id *cm_id,
void *private_data,
uint8_t private_data_len)
{
@@ -367,16 +491,15 @@ int ib_cm_send_drep(uint32_t cm_id,
private_data, private_data_len);
}
-int ib_cm_establish(uint32_t cm_id)
+int ib_cm_establish(struct ib_cm_id *cm_id)
{
struct cm_abi_establish *cmd;
void *msg;
int result;
int size;
- CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_ESTABLISH, size);
-
- cmd->id = cm_id;
+ CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_ESTABLISH, size);
+ cmd->id = cm_id->handle;
result = write(fd, msg, size);
if (result != size)
@@ -385,7 +508,7 @@ int ib_cm_establish(uint32_t cm_id)
return 0;
}
-static inline int cm_send_status(uint32_t cm_id,
+static inline int cm_send_status(struct ib_cm_id *cm_id,
uint32_t type,
int status,
void *info,
@@ -399,19 +522,16 @@ static inline int cm_send_status(uint32_
int size;
CM_CREATE_MSG_CMD(msg, cmd, type, size);
-
- cmd->id = cm_id;
+ cmd->id = cm_id->handle;
cmd->status = status;
if (private_data && private_data_len) {
-
- cmd->data = (unsigned long)private_data;
+ cmd->data = (uintptr_t) private_data;
cmd->data_len = private_data_len;
}
if (info && info_length) {
-
- cmd->info = (unsigned long)info;
+ cmd->info = (uintptr_t) info;
cmd->info_len = info_length;
}
@@ -422,7 +542,7 @@ static inline int cm_send_status(uint32_
return 0;
}
-int ib_cm_send_rej(uint32_t cm_id,
+int ib_cm_send_rej(struct ib_cm_id *cm_id,
enum ib_cm_rej_reason reason,
void *ari,
uint8_t ari_length,
@@ -434,7 +554,7 @@ int ib_cm_send_rej(uint32_t cm_id,
private_data, private_data_len);
}
-int ib_cm_send_apr(uint32_t cm_id,
+int ib_cm_send_apr(struct ib_cm_id *cm_id,
enum ib_cm_apr_status status,
void *info,
uint8_t info_length,
@@ -446,7 +566,7 @@ int ib_cm_send_apr(uint32_t cm_id,
private_data, private_data_len);
}
-int ib_cm_send_mra(uint32_t cm_id,
+int ib_cm_send_mra(struct ib_cm_id *cm_id,
uint8_t service_timeout,
void *private_data,
uint8_t private_data_len)
@@ -457,13 +577,11 @@ int ib_cm_send_mra(uint32_t cm_id,
int size;
CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_SEND_MRA, size);
-
- cmd->id = cm_id;
+ cmd->id = cm_id->handle;
cmd->timeout = service_timeout;
if (private_data && private_data_len) {
-
- cmd->data = (unsigned long)private_data;
+ cmd->data = (uintptr_t) private_data;
cmd->len = private_data_len;
}
@@ -474,7 +592,7 @@ int ib_cm_send_mra(uint32_t cm_id,
return 0;
}
-int ib_cm_send_lap(uint32_t cm_id,
+int ib_cm_send_lap(struct ib_cm_id *cm_id,
struct ib_sa_path_rec *alternate_path,
void *private_data,
uint8_t private_data_len)
@@ -486,22 +604,19 @@ int ib_cm_send_lap(uint32_t cm_id,
int size;
CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_SEND_LAP, size);
-
- cmd->id = cm_id;
+ cmd->id = cm_id->handle;
if (alternate_path) {
-
abi_path = alloca(sizeof(*abi_path));
if (!abi_path)
return -ENOMEM;
cm_param_path_get(abi_path, alternate_path);
- cmd->path = (unsigned long)abi_path;
+ cmd->path = (uintptr_t) abi_path;
}
if (private_data && private_data_len) {
-
- cmd->data = (unsigned long)private_data;
+ cmd->data = (uintptr_t) private_data;
cmd->len = private_data_len;
}
@@ -512,7 +627,8 @@ int ib_cm_send_lap(uint32_t cm_id,
return 0;
}
-int ib_cm_send_sidr_req(uint32_t cm_id, struct ib_cm_sidr_req_param *param)
+int ib_cm_send_sidr_req(struct ib_cm_id *cm_id,
+ struct ib_cm_sidr_req_param *param)
{
struct cm_abi_path_rec *abi_path;
struct cm_abi_sidr_req *cmd;
@@ -524,26 +640,23 @@ int ib_cm_send_sidr_req(uint32_t cm_id,
return -EINVAL;
CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_SEND_SIDR_REQ, size);
-
- cmd->id = cm_id;
+ cmd->id = cm_id->handle;
cmd->sid = param->service_id;
cmd->timeout = param->timeout_ms;
cmd->pkey = param->pkey;
cmd->max_cm_retries = param->max_cm_retries;
if (param->path) {
-
abi_path = alloca(sizeof(*abi_path));
if (!abi_path)
return -ENOMEM;
cm_param_path_get(abi_path, param->path);
- cmd->path = (unsigned long)abi_path;
+ cmd->path = (uintptr_t) abi_path;
}
if (param->private_data && param->private_data_len) {
-
- cmd->data = (unsigned long)param->private_data;
+ cmd->data = (uintptr_t) param->private_data;
cmd->len = param->private_data_len;
}
@@ -554,7 +667,8 @@ int ib_cm_send_sidr_req(uint32_t cm_id,
return 0;
}
-int ib_cm_send_sidr_rep(uint32_t cm_id, struct ib_cm_sidr_rep_param *param)
+int ib_cm_send_sidr_rep(struct ib_cm_id *cm_id,
+ struct ib_cm_sidr_rep_param *param)
{
struct cm_abi_sidr_rep *cmd;
void *msg;
@@ -565,21 +679,18 @@ int ib_cm_send_sidr_rep(uint32_t cm_id,
return -EINVAL;
CM_CREATE_MSG_CMD(msg, cmd, IB_USER_CM_CMD_SEND_SIDR_REP, size);
-
- cmd->id = cm_id;
+ cmd->id = cm_id->handle;
cmd->qpn = param->qp_num;
cmd->qkey = param->qkey;
cmd->status = param->status;
if (param->private_data && param->private_data_len) {
-
- cmd->data = (unsigned long)param->private_data;
+ cmd->data = (uintptr_t) param->private_data;
cmd->data_len = param->private_data_len;
}
if (param->info && param->info_length) {
-
- cmd->info = (unsigned long)param->info;
+ cmd->info = (uintptr_t) param->info;
cmd->info_len = param->info_length;
}
@@ -599,8 +710,8 @@ static void cm_event_path_get(struct ib_
if (!kpath || !upath)
return;
- memcpy(upath->dgid.raw, kpath->dgid, sizeof(union ibv_gid));
- memcpy(upath->sgid.raw, kpath->sgid, sizeof(union ibv_gid));
+ memcpy(upath->dgid.raw, kpath->dgid, sizeof upath->dgid);
+ memcpy(upath->sgid.raw, kpath->sgid, sizeof upath->sgid);
upath->dlid = kpath->dlid;
upath->slid = kpath->slid;
@@ -626,8 +737,6 @@ static void cm_event_path_get(struct ib_
static void cm_event_req_get(struct ib_cm_req_event_param *ureq,
struct cm_abi_req_event_resp *kreq)
{
- ureq->listen_id = kreq->listen_id;
-
ureq->remote_ca_guid = kreq->remote_ca_guid;
ureq->remote_qkey = kreq->remote_qkey;
ureq->remote_qpn = kreq->remote_qpn;
@@ -661,36 +770,6 @@ static void cm_event_rep_get(struct ib_c
urep->rnr_retry_count = krep->rnr_retry_count;
urep->srq = krep->srq;
}
-static void cm_event_rej_get(struct ib_cm_rej_event_param *urej,
- struct cm_abi_rej_event_resp *krej)
-{
- urej->reason = krej->reason;
-}
-
-static void cm_event_mra_get(struct ib_cm_mra_event_param *umra,
- struct cm_abi_mra_event_resp *kmra)
-{
- umra->service_timeout = kmra->timeout;
-}
-
-static void cm_event_lap_get(struct ib_cm_lap_event_param *ulap,
- struct cm_abi_lap_event_resp *klap)
-{
- cm_event_path_get(ulap->alternate_path, &klap->path);
-}
-
-static void cm_event_apr_get(struct ib_cm_apr_event_param *uapr,
- struct cm_abi_apr_event_resp *kapr)
-{
- uapr->ap_status = kapr->status;
-}
-
-static void cm_event_sidr_req_get(struct ib_cm_sidr_req_event_param *ureq,
- struct cm_abi_sidr_req_event_resp *kreq)
-{
- ureq->listen_id = kreq->listen_id;
- ureq->pkey = kreq->pkey;
-}
static void cm_event_sidr_rep_get(struct ib_cm_sidr_rep_event_param *urep,
struct cm_abi_sidr_rep_event_resp *krep)
@@ -702,6 +781,7 @@ static void cm_event_sidr_rep_get(struct
int ib_cm_event_get(struct ib_cm_event **event)
{
+ struct cm_id_private *cm_id_priv;
struct cm_abi_cmd_hdr *hdr;
struct cm_abi_event_get *cmd;
struct cm_abi_event_resp *resp;
@@ -733,7 +813,7 @@ int ib_cm_event_get(struct ib_cm_event *
if (!resp)
return -ENOMEM;
- cmd->response = (unsigned long)resp;
+ cmd->response = (uintptr_t) resp;
cmd->data_len = (uint8_t)(~0U);
cmd->info_len = (uint8_t)(~0U);
@@ -749,8 +829,8 @@ int ib_cm_event_get(struct ib_cm_event *
goto done;
}
- cmd->data = (unsigned long)data;
- cmd->info = (unsigned long)info;
+ cmd->data = (uintptr_t) data;
+ cmd->info = (uintptr_t) info;
result = write(fd, msg, size);
if (result != size) {
@@ -765,14 +845,11 @@ int ib_cm_event_get(struct ib_cm_event *
result = -ENOMEM;
goto done;
}
-
memset(evt, 0, sizeof(*evt));
-
- evt->cm_id = resp->id;
+ evt->cm_id = (void *) (uintptr_t) resp->uid;
evt->event = resp->event;
if (resp->present & CM_ABI_PRES_PRIMARY) {
-
path_a = malloc(sizeof(*path_a));
if (!path_a) {
result = -ENOMEM;
@@ -781,81 +858,78 @@ int ib_cm_event_get(struct ib_cm_event *
}
if (resp->present & CM_ABI_PRES_ALTERNATE) {
-
path_b = malloc(sizeof(*path_b));
if (!path_b) {
result = -ENOMEM;
goto done;
}
}
-
- if (resp->present & CM_ABI_PRES_DATA) {
-
- evt->private_data = data;
- data = NULL;
- }
switch (evt->event) {
case IB_CM_REQ_RECEIVED:
-
+ evt->param.req_rcvd.listen_id = evt->cm_id;
+ cm_id_priv = ib_cm_alloc_id(evt->cm_id->context);
+ if (!cm_id_priv) {
+ result = -ENOMEM;
+ goto done;
+ }
+ cm_id_priv->id.handle = resp->id;
+ evt->cm_id = &cm_id_priv->id;
evt->param.req_rcvd.primary_path = path_a;
evt->param.req_rcvd.alternate_path = path_b;
path_a = NULL;
path_b = NULL;
-
cm_event_req_get(&evt->param.req_rcvd, &resp->u.req_resp);
break;
case IB_CM_REP_RECEIVED:
-
cm_event_rep_get(&evt->param.rep_rcvd, &resp->u.rep_resp);
break;
case IB_CM_MRA_RECEIVED:
-
- cm_event_mra_get(&evt->param.mra_rcvd, &resp->u.mra_resp);
+ evt->param.mra_rcvd.service_timeout = resp->u.mra_resp.timeout;
break;
case IB_CM_REJ_RECEIVED:
-
- cm_event_rej_get(&evt->param.rej_rcvd, &resp->u.rej_resp);
-
+ evt->param.rej_rcvd.reason = resp->u.rej_resp.reason;
evt->param.rej_rcvd.ari = info;
info = NULL;
-
break;
case IB_CM_LAP_RECEIVED:
-
evt->param.lap_rcvd.alternate_path = path_b;
path_b = NULL;
-
- cm_event_lap_get(&evt->param.lap_rcvd, &resp->u.lap_resp);
+ cm_event_path_get(evt->param.lap_rcvd.alternate_path,
+ &resp->u.lap_resp.path);
break;
case IB_CM_APR_RECEIVED:
-
- cm_event_apr_get(&evt->param.apr_rcvd, &resp->u.apr_resp);
-
+ evt->param.apr_rcvd.ap_status = resp->u.apr_resp.status;
evt->param.apr_rcvd.apr_info = info;
info = NULL;
-
break;
case IB_CM_SIDR_REQ_RECEIVED:
-
- cm_event_sidr_req_get(&evt->param.sidr_req_rcvd,
- &resp->u.sidr_req_resp);
+ evt->param.sidr_req_rcvd.listen_id = evt->cm_id;
+ cm_id_priv = ib_cm_alloc_id(evt->cm_id->context);
+ if (!cm_id_priv) {
+ result = -ENOMEM;
+ goto done;
+ }
+ cm_id_priv->id.handle = resp->id;
+ evt->cm_id = &cm_id_priv->id;
+ evt->param.sidr_req_rcvd.pkey = resp->u.sidr_req_resp.pkey;
break;
case IB_CM_SIDR_REP_RECEIVED:
-
cm_event_sidr_rep_get(&evt->param.sidr_rep_rcvd,
&resp->u.sidr_rep_resp);
-
evt->param.sidr_rep_rcvd.info = info;
info = NULL;
-
break;
default:
-
evt->param.send_status = resp->u.send_status;
break;
}
+ if (resp->present & CM_ABI_PRES_DATA) {
+ evt->private_data = data;
+ data = NULL;
+ }
+
*event = evt;
evt = NULL;
result = 0;
@@ -876,44 +950,51 @@ done:
int ib_cm_event_put(struct ib_cm_event *event)
{
+ struct cm_id_private *cm_id_priv;
+
if (!event)
return -EINVAL;
if (event->private_data)
free(event->private_data);
+ cm_id_priv = container_of(event->cm_id, struct cm_id_private, id);
+
switch (event->event) {
case IB_CM_REQ_RECEIVED:
-
- if (event->param.req_rcvd.primary_path)
- free(event->param.req_rcvd.primary_path);
-
+ cm_id_priv = container_of(event->param.req_rcvd.listen_id,
+ struct cm_id_private, id);
+ free(event->param.req_rcvd.primary_path);
if (event->param.req_rcvd.alternate_path)
free(event->param.req_rcvd.alternate_path);
break;
case IB_CM_REJ_RECEIVED:
-
if (event->param.rej_rcvd.ari)
free(event->param.rej_rcvd.ari);
break;
case IB_CM_LAP_RECEIVED:
-
- if (event->param.lap_rcvd.alternate_path)
- free(event->param.lap_rcvd.alternate_path);
+ free(event->param.lap_rcvd.alternate_path);
break;
case IB_CM_APR_RECEIVED:
-
if (event->param.apr_rcvd.apr_info)
free(event->param.apr_rcvd.apr_info);
break;
+ case IB_CM_SIDR_REQ_RECEIVED:
+ cm_id_priv = container_of(event->param.sidr_req_rcvd.listen_id,
+ struct cm_id_private, id);
+ break;
case IB_CM_SIDR_REP_RECEIVED:
-
if (event->param.sidr_rep_rcvd.info)
free(event->param.sidr_rep_rcvd.info);
default:
break;
}
+ pthread_mutex_lock(&cm_id_priv->mut);
+ cm_id_priv->events_completed++;
+ pthread_cond_signal(&cm_id_priv->cond);
+ pthread_mutex_unlock(&cm_id_priv->mut);
+
free(event);
return 0;
}
Index: userspace/libibcm/Makefile.am
===================================================================
--- userspace/libibcm/Makefile.am (revision 3124)
+++ userspace/libibcm/Makefile.am (working copy)
@@ -18,9 +18,11 @@ endif
src_libibcm_la_SOURCES = src/cm.c
src_libibcm_la_LDFLAGS = -avoid-version -module $(ucm_version_script)
-bin_PROGRAMS = examples/ucm_simple
-examples_ucm_simple_SOURCES = examples/simple.c
-examples_ucm_simple_LDADD = $(top_builddir)/src/libibcm.la
+bin_PROGRAMS = examples/ucmpost
+examples_ucmpost_SOURCES = examples/cmpost.c
+examples_ucmpost_LDADD = $(top_builddir)/src/libibcm.la \
+ $(libdir)/libibverbs.la \
+ $(libdir)/libibat.la
libibcmincludedir = $(includedir)/infiniband
Index: userspace/libibcm/examples/cmpost.c
===================================================================
--- userspace/libibcm/examples/cmpost.c (revision 0)
+++ userspace/libibcm/examples/cmpost.c (revision 0)
@@ -0,0 +1,718 @@
+/*
+ * Copyright (c) 2004, 2005 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <byteswap.h>
+
+#include <infiniband/cm.h>
+#include <infiniband/at.h>
+
+#if __BYTE_ORDER == __BIG_ENDIAN
+static inline uint64_t cpu_to_be64(uint64_t x) { return x; }
+static inline uint32_t cpu_to_be32(uint32_t x) { return x; }
+#else
+static inline uint64_t cpu_to_be64(uint64_t x) { return bswap_64(x); }
+static inline uint32_t cpu_to_be32(uint32_t x) { return bswap_32(x); }
+#endif
+
+/*
+ * To execute:
+ * Server: ucmpost
+ * Client: ucmpost server
+ */
+
+struct cmtest {
+ struct ibv_device *device;
+ struct ibv_context *verbs;
+ struct ibv_pd *pd;
+
+ /* cm info */
+ struct ib_sa_path_rec path_rec;
+
+ struct cmtest_node *nodes;
+ int conn_index;
+ int connects_left;
+ int disconnects_left;
+
+ /* memory region info */
+ struct ibv_mr *mr;
+ void *mem;
+};
+
+static struct cmtest test;
+static int message_count = 10;
+static int message_size = 100;
+static int connections = 1;
+static int is_server = 1;
+
+struct cmtest_node {
+ int id;
+ struct ibv_cq *cq;
+ struct ibv_qp *qp;
+ struct ib_cm_id *cm_id;
+ int connected;
+};
+
+static int post_recvs(struct cmtest_node *node)
+{
+ struct ibv_recv_wr recv_wr, *recv_failure;
+ struct ibv_sge sge;
+ int i, ret = 0;
+
+ if (!message_count)
+ return 0;
+
+ recv_wr.next = NULL;
+ recv_wr.sg_list = &sge;
+ recv_wr.num_sge = 1;
+ recv_wr.wr_id = (uintptr_t) node;
+
+ sge.length = message_size;
+ sge.lkey = test.mr->lkey;
+ sge.addr = (uintptr_t) test.mem;
+
+ for (i = 0; i < message_count && !ret; i++ ) {
+ ret = ibv_post_recv(node->qp, &recv_wr, &recv_failure);
+ if (ret) {
+ printf("failed to post receives: %d\n", ret);
+ break;
+ }
+ }
+ return ret;
+}
+
+static int modify_to_rtr(struct cmtest_node *node)
+{
+ struct ibv_qp_attr qp_attr;
+ int qp_attr_mask, ret;
+
+ qp_attr.qp_state = IBV_QPS_INIT;
+ ret = ib_cm_init_qp_attr(node->cm_id, &qp_attr, &qp_attr_mask);
+ if (ret) {
+ printf("failed to init QP attr for INIT: %d\n", ret);
+ return ret;
+ }
+ ret = ibv_modify_qp(node->qp, &qp_attr, qp_attr_mask);
+ if (ret) {
+ printf("failed to modify QP to INIT: %d\n", ret);
+ return ret;
+ }
+ qp_attr.qp_state = IBV_QPS_RTR;
+ ret = ib_cm_init_qp_attr(node->cm_id, &qp_attr, &qp_attr_mask);
+ if (ret) {
+ printf("failed to init QP attr for RTR: %d\n", ret);
+ return ret;
+ }
+ qp_attr.rq_psn = node->qp->qp_num;
+ ret = ibv_modify_qp(node->qp, &qp_attr, qp_attr_mask);
+ if (ret) {
+ printf("failed to modify QP to RTR: %d\n", ret);
+ return ret;
+ }
+ return 0;
+}
+
+static int modify_to_rts(struct cmtest_node *node)
+{
+ struct ibv_qp_attr qp_attr;
+ int qp_attr_mask, ret;
+
+ qp_attr.qp_state = IBV_QPS_RTS;
+ ret = ib_cm_init_qp_attr(node->cm_id, &qp_attr, &qp_attr_mask);
+ if (ret) {
+ printf("failed to init QP attr for RTS: %d\n", ret);
+ return ret;
+ }
+ ret = ibv_modify_qp(node->qp, &qp_attr, qp_attr_mask);
+ if (ret) {
+ printf("failed to modify QP to RTS: %d\n", ret);
+ return ret;
+ }
+ return 0;
+}
+
+static void req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
+{
+ struct cmtest_node *node;
+ struct ib_cm_req_event_param *req;
+ struct ib_cm_rep_param rep;
+ int ret;
+
+ if (test.conn_index == connections)
+ goto error1;
+ node = &test.nodes[test.conn_index++];
+
+ node->cm_id = cm_id;
+ cm_id->context = node;
+
+ ret = modify_to_rtr(node);
+ if (ret)
+ goto error2;
+
+ ret = post_recvs(node);
+ if (ret)
+ goto error2;
+
+ req = &event->param.req_rcvd;
+ memset(&rep, 0, sizeof rep);
+ rep.qp_num = node->qp->qp_num;
+ rep.srq = (node->qp->srq != NULL);
+ rep.starting_psn = node->qp->qp_num;
+ rep.responder_resources = req->responder_resources;
+ rep.initiator_depth = req->initiator_depth;
+ rep.target_ack_delay = 20;
+ rep.flow_control = req->flow_control;
+ rep.rnr_retry_count = req->rnr_retry_count;
+
+ ret = ib_cm_send_rep(cm_id, &rep);
+ if (ret) {
+ printf("failed to send CM REP: %d\n", ret);
+ goto error2;
+ }
+ return;
+error2:
+ test.disconnects_left--;
+ test.connects_left--;
+error1:
+ printf("failing connection request\n");
+ ib_cm_send_rej(cm_id, IB_CM_REJ_UNSUPPORTED, NULL, 0, NULL, 0);
+}
+
+static void rep_handler(struct cmtest_node *node, struct ib_cm_event *event)
+{
+ int ret;
+
+ ret = modify_to_rtr(node);
+ if (ret)
+ goto error;
+
+ ret = modify_to_rts(node);
+ if (ret)
+ goto error;
+
+ ret = post_recvs(node);
+ if (ret)
+ goto error;
+
+ ret = ib_cm_send_rtu(node->cm_id, NULL, 0);
+ if (ret) {
+ printf("failed to send CM RTU: %d\n", ret);
+ goto error;
+ }
+ node->connected = 1;
+ test.connects_left--;
+ return;
+error:
+ printf("failing connection reply\n");
+ ib_cm_send_rej(node->cm_id, IB_CM_REJ_UNSUPPORTED, NULL, 0, NULL, 0);
+ test.disconnects_left--;
+ test.connects_left--;
+}
+
+static void rtu_handler(struct cmtest_node *node)
+{
+ int ret;
+
+ ret = modify_to_rts(node);
+ if (ret)
+ goto error;
+
+ node->connected = 1;
+ test.connects_left--;
+ return;
+error:
+ printf("aborting connection - disconnecting\n");
+ ib_cm_send_dreq(node->cm_id, NULL, 0);
+ test.disconnects_left--;
+ test.connects_left--;
+}
+
+static void cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
+{
+ struct cmtest_node *node = cm_id->context;
+
+ switch (event->event) {
+ case IB_CM_REQ_RECEIVED:
+ req_handler(cm_id, event);
+ break;
+ case IB_CM_REP_RECEIVED:
+ rep_handler(node, event);
+ break;
+ case IB_CM_RTU_RECEIVED:
+ rtu_handler(node);
+ break;
+ case IB_CM_DREQ_RECEIVED:
+ node->connected = 0;
+ ib_cm_send_drep(node->cm_id, NULL, 0);
+ test.disconnects_left--;
+ break;
+ case IB_CM_DREP_RECEIVED:
+ test.disconnects_left--;
+ break;
+ case IB_CM_REJ_RECEIVED:
+ printf("Received REJ\n");
+ /* fall through */
+ case IB_CM_REQ_ERROR:
+ case IB_CM_REP_ERROR:
+ printf("Error sending REQ or REP\n");
+ test.disconnects_left--;
+ test.connects_left--;
+ break;
+ case IB_CM_DREQ_ERROR:
+ test.disconnects_left--;
+ printf("Error sending DREQ\n");
+ break;
+ default:
+ break;
+ }
+}
+
+static int init_node(struct cmtest_node *node, struct ibv_qp_init_attr *qp_attr)
+{
+ int cqe, ret;
+
+ if (!is_server) {
+ ret = ib_cm_create_id(&node->cm_id, node);
+ if (ret) {
+ printf("failed to create cm_id: %d\n", ret);
+ return ret;
+ }
+ }
+
+ cqe = message_count ? message_count * 2 : 2;
+ node->cq = ibv_create_cq(test.verbs, cqe, node);
+ if (!node->cq) {
+ printf("unable to create CQ\n");
+ goto error1;
+ }
+
+ qp_attr->send_cq = node->cq;
+ qp_attr->recv_cq = node->cq;
+ node->qp = ibv_create_qp(test.pd, qp_attr);
+ if (!node->qp) {
+ printf("unable to create QP\n");
+ goto error2;
+ }
+ return 0;
+error2:
+ ibv_destroy_cq(node->cq);
+error1:
+ if (!is_server)
+ ib_cm_destroy_id(node->cm_id);
+ return -1;
+}
+
+static void destroy_node(struct cmtest_node *node)
+{
+ ibv_destroy_qp(node->qp);
+ ibv_destroy_cq(node->cq);
+ if (node->cm_id)
+ ib_cm_destroy_id(node->cm_id);
+}
+
+static int create_nodes(void)
+{
+ struct ibv_qp_init_attr qp_attr;
+ int ret, i;
+
+ test.nodes = malloc(sizeof *test.nodes * connections);
+ if (!test.nodes) {
+ printf("unable to allocate memory for test nodes\n");
+ return -1;
+ }
+ memset(test.nodes, 0, sizeof *test.nodes * connections);
+
+ memset(&qp_attr, 0, sizeof qp_attr);
+ qp_attr.cap.max_send_wr = message_count ? message_count : 1;
+ qp_attr.cap.max_recv_wr = message_count ? message_count : 1;
+ qp_attr.cap.max_send_sge = 1;
+ qp_attr.cap.max_recv_sge = 1;
+ qp_attr.qp_type = IBV_QPT_RC;
+
+ for (i = 0; i < connections; i++) {
+ test.nodes[i].id = i;
+ ret = init_node(&test.nodes[i], &qp_attr);
+ if (ret)
+ goto error;
+ }
+ return 0;
+error:
+ while (--i >= 0)
+ destroy_node(&test.nodes[i]);
+ free(test.nodes);
+ return ret;
+}
+
+static void destroy_nodes(void)
+{
+ int i;
+
+ for (i = 0; i < connections; i++)
+ destroy_node(&test.nodes[i]);
+ free(test.nodes);
+}
+
+static int create_messages(void)
+{
+ if (!message_size)
+ message_count = 0;
+
+ if (!message_count)
+ return 0;
+
+ test.mem = malloc(message_size);
+ if (!test.mem) {
+ printf("failed message allocation\n");
+ return -1;
+ }
+ test.mr = ibv_reg_mr(test.pd, test.mem, message_size,
+ IBV_ACCESS_LOCAL_WRITE);
+ if (!test.mr) {
+ printf("failed to reg MR\n");
+ goto err;
+ }
+ return 0;
+err:
+ free(test.mem);
+ return -1;
+}
+
+static void destroy_messages(void)
+{
+ if (!message_count)
+ return;
+
+ ibv_dereg_mr(test.mr);
+ free(test.mem);
+}
+
+static int init(void)
+{
+ struct dlist *dev_list;
+ int ret;
+
+ test.connects_left = connections;
+ test.disconnects_left = connections;
+
+ dev_list = ibv_get_devices();
+ dlist_start(dev_list);
+ test.device = dlist_next(dev_list);
+ if (!test.device)
+ return -1;
+
+ test.verbs = ibv_open_device(test.device);
+ if (!test.verbs)
+ return -1;
+
+ test.pd = ibv_alloc_pd(test.verbs);
+ if (!test.pd) {
+ printf("failed to alloc PD\n");
+ return -1;
+ }
+ ret = create_messages();
+ if (ret) {
+ printf("unable to create test messages\n");
+ goto error1;
+ }
+ ret = create_nodes();
+ if (ret) {
+ printf("unable to create test nodes\n");
+ goto error2;
+ }
+ return 0;
+error2:
+ destroy_messages();
+error1:
+ ibv_dealloc_pd(test.pd);
+ return -1;
+}
+
+static void cleanup(void)
+{
+ destroy_nodes();
+ destroy_messages();
+ ibv_dealloc_pd(test.pd);
+}
+
+static int send_msgs(void)
+{
+ struct ibv_send_wr send_wr, *bad_send_wr;
+ struct ibv_sge sge;
+ int i, m, ret;
+
+ send_wr.next = NULL;
+ send_wr.sg_list = &sge;
+ send_wr.num_sge = 1;
+ send_wr.opcode = IBV_WR_SEND;
+ send_wr.send_flags = IBV_SEND_SIGNALED;
+ send_wr.wr_id = 0;
+
+ sge.addr = (uintptr_t) test.mem;
+ sge.length = message_size;
+ sge.lkey = test.mr->lkey;
+
+ for (i = 0; i < connections; i++) {
+ if (!test.nodes[i].connected)
+ continue;
+
+ for (m = 0; m < message_count; m++) {
+ ret = ibv_post_send(test.nodes[i].qp, &send_wr,
+ &bad_send_wr);
+ if (ret)
+ return ret;
+ }
+ }
+ return 0;
+}
+
+static int poll_cqs(void)
+{
+ struct ibv_wc wc[8];
+ int done, i, ret;
+
+ for (i = 0; i < connections; i++) {
+ if (!test.nodes[i].connected)
+ continue;
+
+ for (done = 0; done < message_count; done += ret) {
+ ret = ibv_poll_cq(test.nodes[i].cq, 8, wc);
+ if (ret < 0) {
+ printf("failed polling CQ: %d\n", ret);
+ return ret;
+ }
+ }
+ }
+ return 0;
+}
+
+static void connect_events(void)
+{
+ struct ib_cm_event *event;
+ int err = 0;
+
+ while (test.connects_left && !err) {
+ err = ib_cm_event_get(&event);
+ if (!err) {
+ cm_handler(event->cm_id, event);
+ ib_cm_event_put(event);
+ }
+ }
+}
+
+static void disconnect_events(void)
+{
+ struct ib_cm_event *event;
+ int err = 0;
+
+ while (test.disconnects_left && !err) {
+ err = ib_cm_event_get(&event);
+ if (!err) {
+ cm_handler(event->cm_id, event);
+ ib_cm_event_put(event);
+ }
+ }
+}
+
+static void run_server(void)
+{
+ struct ib_cm_id *listen_id;
+ int i, ret;
+
+ printf("starting server\n");
+ if (ib_cm_create_id(&listen_id, &test)) {
+ printf("listen request failed\n");
+ return;
+ }
+ ret = ib_cm_listen(listen_id, cpu_to_be64(0x1000), 0);
+ if (ret) {
+ printf("failure trying to listen: %d\n", ret);
+ goto out;
+ }
+
+ connect_events();
+
+ if (message_count) {
+ printf("initiating data transfers\n");
+ if (send_msgs())
+ goto out;
+ printf("receiving data transfers\n");
+ if (poll_cqs())
+ goto out;
+ printf("data transfers complete\n");
+ }
+
+ printf("disconnecting\n");
+ for (i = 0; i < connections; i++) {
+ if (!test.nodes[i].connected)
+ continue;
+
+ test.nodes[i].connected = 0;
+ ib_cm_send_dreq(test.nodes[i].cm_id, NULL, 0);
+ }
+ disconnect_events();
+ printf("disconnected\n");
+out:
+ ib_cm_destroy_id(listen_id);
+}
+
+static void at_callback(uint64_t req_id, void *context, int rec_num)
+{
+}
+
+static int query_for_path(char *dest)
+{
+ struct ib_at_ib_route route;
+ struct ib_at_completion comp;
+ struct addrinfo *res;
+ int ret;
+
+ ret = getaddrinfo(dest, NULL, NULL, &res);
+ if (ret) {
+ printf("getaddrinfo failed - invalid hostname or IP address\n");
+ return ret;
+ }
+
+ if (res->ai_family != PF_INET) {
+ ret = -1;
+ goto out;
+ }
+
+ comp.fn = at_callback;
+ ret = ib_at_route_by_ip(((struct sockaddr_in *)res->ai_addr)->sin_addr.s_addr,
+ 0, 0, 0, &route, &comp, NULL);
+ if (ret < 0) {
+ printf("ib_at_route_by_ip failed: %d\n", ret);
+ goto out;
+ }
+
+ if (!ret) {
+ ret = ib_at_callback_get();
+ if (ret) {
+ printf("ib_at_callback_get failed: %d\n", ret);
+ goto out;
+ }
+ }
+
+ ret = ib_at_paths_by_route(&route, 0, &test.path_rec, 1, &comp, NULL);
+ if (ret < 0) {
+ printf("ib_at_paths_by_route failed: %d\n", ret);
+ goto out;
+ }
+
+ if (!ret) {
+ ret = ib_at_callback_get();
+ if (ret)
+ printf("ib_at_callback_get failed: %d\n", ret);
+ } else
+ ret = 0;
+
+out:
+ freeaddrinfo(res);
+ return ret;
+}
+
+static void run_client(char *dest)
+{
+ struct ib_cm_req_param req;
+ int i, ret;
+
+ printf("starting client\n");
+ ret = query_for_path(dest);
+ if (ret) {
+ printf("failed path record query: %d\n", ret);
+ return;
+ }
+
+ memset(&req, 0, sizeof req);
+ req.primary_path = &test.path_rec;
+ req.service_id = cpu_to_be64(0x1000);
+ req.responder_resources = 1;
+ req.initiator_depth = 1;
+ req.remote_cm_response_timeout = 20;
+ req.local_cm_response_timeout = 20;
+ req.retry_count = 5;
+ req.max_cm_retries = 5;
+
+ printf("connecting\n");
+ for (i = 0; i < connections; i++) {
+ req.qp_num = test.nodes[i].qp->qp_num;
+ req.qp_type = IBV_QPT_RC;
+ req.srq = (test.nodes[i].qp->srq != NULL);
+ req.starting_psn = test.nodes[i].qp->qp_num;
+ ret = ib_cm_send_req(test.nodes[i].cm_id, &req);
+ if (ret) {
+ printf("failure sending REQ: %d\n", ret);
+ return;
+ }
+ }
+
+ connect_events();
+
+ if (message_count) {
+ printf("receiving data transfers\n");
+ if (poll_cqs())
+ goto out;
+ printf("initiating data transfers\n");
+ if (send_msgs())
+ goto out;
+ printf("data transfers complete\n");
+ }
+out:
+ disconnect_events();
+}
+
+int main(int argc, char **argv)
+{
+ if (argc != 1 && argc != 2) {
+ printf("usage: %s [server_addr]\n", argv[0]);
+ exit(1);
+ }
+
+ is_server = (argc == 1);
+ if (init())
+ exit(1);
+
+ if (is_server)
+ run_server();
+ else
+ run_client(argv[1]);
+
+ printf("test complete\n");
+ cleanup();
+ return 0;
+}
Index: userspace/libibcm/examples/simple.c
===================================================================
--- userspace/libibcm/examples/simple.c (revision 3124)
+++ userspace/libibcm/examples/simple.c (working copy)
@@ -58,7 +58,7 @@ static inline uint64_t cpu_to_be64(uint6
#define TEST_SID 0x0000000ff0000000ULL
-static int cm_connect(uint32_t cm_id)
+static int cm_connect(struct ib_cm_id *cm_id)
{
struct ib_cm_req_param param;
struct ib_sa_path_rec sa;
@@ -108,8 +108,8 @@ static int cm_connect(uint32_t cm_id)
src->global.subnet_prefix = cpu_to_be64(0xfe80000000000000ULL);
dst->global.subnet_prefix = cpu_to_be64(0xfe80000000000000ULL);
- src->global.interface_id = cpu_to_be64(0x0002c90200002179ULL);
- dst->global.interface_id = cpu_to_be64(0x0005ad000001296cULL);
+ src->global.interface_id = cpu_to_be64(0x0002c90107fc5e11ULL);
+ dst->global.interface_id = cpu_to_be64(0x0002c90107fc5eb1ULL);
return ib_cm_send_req(cm_id, ¶m);
}
@@ -118,7 +118,7 @@ int main(int argc, char **argv)
{
struct ib_cm_event *event;
struct ib_cm_rep_param rep;
- int cm_id;
+ struct ib_cm_id *cm_id;
int result;
int param_c = 0;
@@ -137,8 +137,8 @@ int main(int argc, char **argv)
exit(1);
}
- result = ib_cm_create_id(&cm_id);
- if (result < 0) {
+ result = ib_cm_create_id(&cm_id, NULL);
+ if (result) {
printf("Error creating CM ID <%d:%d>\n", result, errno);
goto done;
}
@@ -146,16 +146,16 @@ int main(int argc, char **argv)
if (mode) {
result = cm_connect(cm_id);
if (result) {
- printf("Error <%d:%d> sending REQ <%d>\n",
- result, errno, cm_id);
+ printf("Error <%d:%d> sending REQ\n",
+ result, errno);
goto done;
}
}
else {
result = ib_cm_listen(cm_id, TEST_SID, 0);
if (result) {
- printf("Error <%d:%d> listening <%d>\n",
- result, errno, cm_id);
+ printf("Error <%d:%d> listening\n",
+ result, errno);
goto done;
}
}
@@ -169,7 +169,7 @@ int main(int argc, char **argv)
goto done;
}
- printf("CM ID <%d> Event <%d>\n", event->cm_id, event->event);
+ printf("CM ID <%p> Event <%d>\n", event->cm_id, event->event);
switch (event->event) {
case IB_CM_REQ_RECEIVED:
@@ -264,4 +264,3 @@ int main(int argc, char **argv)
done:
return 0;
}
-
Index: linux-kernel/infiniband/include/ib_user_cm.h
===================================================================
--- linux-kernel/infiniband/include/ib_user_cm.h (revision 3109)
+++ linux-kernel/infiniband/include/ib_user_cm.h (working copy)
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2005 Topspin Communications. All rights reserved.
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -37,7 +38,7 @@
#include <linux/types.h>
-#define IB_USER_CM_ABI_VERSION 1
+#define IB_USER_CM_ABI_VERSION 2
enum {
IB_USER_CM_CMD_CREATE_ID,
@@ -60,6 +61,7 @@ enum {
IB_USER_CM_CMD_SEND_SIDR_REP,
IB_USER_CM_CMD_EVENT,
+ IB_USER_CM_CMD_INIT_QP_ATTR,
};
/*
* command ABI structures.
@@ -71,6 +73,7 @@ struct ib_ucm_cmd_hdr {
};
struct ib_ucm_create_id {
+ __u64 uid;
__u64 response;
};
@@ -79,9 +82,14 @@ struct ib_ucm_create_id_resp {
};
struct ib_ucm_destroy_id {
+ __u64 response;
__u32 id;
};
+struct ib_ucm_destroy_id_resp {
+ __u32 events_reported;
+};
+
struct ib_ucm_attr_id {
__u64 response;
__u32 id;
@@ -94,6 +102,64 @@ struct ib_ucm_attr_id_resp {
__be32 remote_id;
};
+struct ib_ucm_init_qp_attr {
+ __u64 response;
+ __u32 id;
+ __u32 qp_state;
+};
+
+struct ib_ucm_ah_attr {
+ __u8 grh_dgid[16];
+ __u32 grh_flow_label;
+ __u16 dlid;
+ __u16 reserved;
+ __u8 grh_sgid_index;
+ __u8 grh_hop_limit;
+ __u8 grh_traffic_class;
+ __u8 sl;
+ __u8 src_path_bits;
+ __u8 static_rate;
+ __u8 is_global;
+ __u8 port_num;
+};
+
+struct ib_ucm_init_qp_attr_resp {
+ __u32 qp_attr_mask;
+ __u32 qp_state;
+ __u32 cur_qp_state;
+ __u32 path_mtu;
+ __u32 path_mig_state;
+ __u32 qkey;
+ __u32 rq_psn;
+ __u32 sq_psn;
+ __u32 dest_qp_num;
+ __u32 qp_access_flags;
+
+ struct ib_ucm_ah_attr ah_attr;
+ struct ib_ucm_ah_attr alt_ah_attr;
+
+ /* ib_qp_cap */
+ __u32 max_send_wr;
+ __u32 max_recv_wr;
+ __u32 max_send_sge;
+ __u32 max_recv_sge;
+ __u32 max_inline_data;
+
+ __u16 pkey_index;
+ __u16 alt_pkey_index;
+ __u8 en_sqd_async_notify;
+ __u8 sq_draining;
+ __u8 max_rd_atomic;
+ __u8 max_dest_rd_atomic;
+ __u8 min_rnr_timer;
+ __u8 port_num;
+ __u8 timeout;
+ __u8 retry_cnt;
+ __u8 rnr_retry;
+ __u8 alt_port_num;
+ __u8 alt_timeout;
+};
+
struct ib_ucm_listen {
__be64 service_id;
__be64 service_mask;
@@ -157,6 +223,7 @@ struct ib_ucm_req {
};
struct ib_ucm_rep {
+ __u64 uid;
__u64 data;
__u32 id;
__u32 qpn;
@@ -232,7 +299,6 @@ struct ib_ucm_event_get {
};
struct ib_ucm_req_event_resp {
- __u32 listen_id;
/* device */
/* port */
struct ib_ucm_path_rec primary_path;
@@ -287,7 +353,6 @@ struct ib_ucm_apr_event_resp {
};
struct ib_ucm_sidr_req_event_resp {
- __u32 listen_id;
/* device */
/* port */
__u16 pkey;
@@ -307,6 +372,7 @@ struct ib_ucm_sidr_rep_event_resp {
#define IB_UCM_PRES_ALTERNATE 0x08
struct ib_ucm_event_resp {
+ __u64 uid;
__u32 id;
__u32 event;
__u32 present;
Index: linux-kernel/infiniband/core/ucm.c
===================================================================
--- linux-kernel/infiniband/core/ucm.c (revision 3109)
+++ linux-kernel/infiniband/core/ucm.c (working copy)
@@ -72,7 +72,6 @@ enum {
static struct semaphore ctx_id_mutex;
static struct idr ctx_id_table;
-static int ctx_id_rover = 0;
static struct ib_ucm_context *ib_ucm_ctx_get(struct ib_ucm_file *file, int id)
{
@@ -97,33 +96,16 @@ static void ib_ucm_ctx_put(struct ib_ucm
wake_up(&ctx->wait);
}
-static ssize_t ib_ucm_destroy_ctx(struct ib_ucm_file *file, int id)
+static inline int ib_ucm_new_cm_id(int event)
{
- struct ib_ucm_context *ctx;
- struct ib_ucm_event *uevent;
-
- down(&ctx_id_mutex);
- ctx = idr_find(&ctx_id_table, id);
- if (!ctx)
- ctx = ERR_PTR(-ENOENT);
- else if (ctx->file != file)
- ctx = ERR_PTR(-EINVAL);
- else
- idr_remove(&ctx_id_table, ctx->id);
- up(&ctx_id_mutex);
-
- if (IS_ERR(ctx))
- return PTR_ERR(ctx);
-
- atomic_dec(&ctx->ref);
- wait_event(ctx->wait, !atomic_read(&ctx->ref));
+ return event == IB_CM_REQ_RECEIVED || event == IB_CM_SIDR_REQ_RECEIVED;
+}
- /* No new events will be generated after destroying the cm_id. */
- if (!IS_ERR(ctx->cm_id))
- ib_destroy_cm_id(ctx->cm_id);
+static void ib_ucm_cleanup_events(struct ib_ucm_context *ctx)
+{
+ struct ib_ucm_event *uevent;
- /* Cleanup events not yet reported to the user. */
- down(&file->mutex);
+ down(&ctx->file->mutex);
list_del(&ctx->file_list);
while (!list_empty(&ctx->events)) {
@@ -133,15 +115,12 @@ static ssize_t ib_ucm_destroy_ctx(struct
list_del(&uevent->ctx_list);
/* clear incoming connections. */
- if (uevent->cm_id)
+ if (ib_ucm_new_cm_id(uevent->resp.event))
ib_destroy_cm_id(uevent->cm_id);
kfree(uevent);
}
- up(&file->mutex);
-
- kfree(ctx);
- return 0;
+ up(&ctx->file->mutex);
}
static struct ib_ucm_context *ib_ucm_ctx_alloc(struct ib_ucm_file *file)
@@ -153,36 +132,31 @@ static struct ib_ucm_context *ib_ucm_ctx
if (!ctx)
return NULL;
+ memset(ctx, 0, sizeof *ctx);
atomic_set(&ctx->ref, 1);
init_waitqueue_head(&ctx->wait);
ctx->file = file;
-
INIT_LIST_HEAD(&ctx->events);
- list_add_tail(&ctx->file_list, &file->ctxs);
-
- ctx_id_rover = (ctx_id_rover + 1) & INT_MAX;
-retry:
- result = idr_pre_get(&ctx_id_table, GFP_KERNEL);
- if (!result)
- goto error;
+ do {
+ result = idr_pre_get(&ctx_id_table, GFP_KERNEL);
+ if (!result)
+ goto error;
+
+ down(&ctx_id_mutex);
+ result = idr_get_new(&ctx_id_table, ctx, &ctx->id);
+ up(&ctx_id_mutex);
+ } while (result == -EAGAIN);
- down(&ctx_id_mutex);
- result = idr_get_new_above(&ctx_id_table, ctx, ctx_id_rover, &ctx->id);
- up(&ctx_id_mutex);
-
- if (result == -EAGAIN)
- goto retry;
if (result)
goto error;
+ list_add_tail(&ctx->file_list, &file->ctxs);
ucm_dbg("Allocated CM ID <%d>\n", ctx->id);
-
return ctx;
+
error:
- list_del(&ctx->file_list);
kfree(ctx);
-
return NULL;
}
/*
@@ -219,12 +193,9 @@ static void ib_ucm_event_path_get(struct
kpath->packet_life_time_selector;
}
-static void ib_ucm_event_req_get(struct ib_ucm_context *ctx,
- struct ib_ucm_req_event_resp *ureq,
+static void ib_ucm_event_req_get(struct ib_ucm_req_event_resp *ureq,
struct ib_cm_req_event_param *kreq)
{
- ureq->listen_id = ctx->id;
-
ureq->remote_ca_guid = kreq->remote_ca_guid;
ureq->remote_qkey = kreq->remote_qkey;
ureq->remote_qpn = kreq->remote_qpn;
@@ -259,14 +230,6 @@ static void ib_ucm_event_rep_get(struct
urep->srq = krep->srq;
}
-static void ib_ucm_event_sidr_req_get(struct ib_ucm_context *ctx,
- struct ib_ucm_sidr_req_event_resp *ureq,
- struct ib_cm_sidr_req_event_param *kreq)
-{
- ureq->listen_id = ctx->id;
- ureq->pkey = kreq->pkey;
-}
-
static void ib_ucm_event_sidr_rep_get(struct ib_ucm_sidr_rep_event_resp *urep,
struct ib_cm_sidr_rep_event_param *krep)
{
@@ -275,15 +238,14 @@ static void ib_ucm_event_sidr_rep_get(st
urep->qpn = krep->qpn;
};
-static int ib_ucm_event_process(struct ib_ucm_context *ctx,
- struct ib_cm_event *evt,
+static int ib_ucm_event_process(struct ib_cm_event *evt,
struct ib_ucm_event *uvt)
{
void *info = NULL;
switch (evt->event) {
case IB_CM_REQ_RECEIVED:
- ib_ucm_event_req_get(ctx, &uvt->resp.u.req_resp,
+ ib_ucm_event_req_get(&uvt->resp.u.req_resp,
&evt->param.req_rcvd);
uvt->data_len = IB_CM_REQ_PRIVATE_DATA_SIZE;
uvt->resp.present = IB_UCM_PRES_PRIMARY;
@@ -331,8 +293,8 @@ static int ib_ucm_event_process(struct i
info = evt->param.apr_rcvd.apr_info;
break;
case IB_CM_SIDR_REQ_RECEIVED:
- ib_ucm_event_sidr_req_get(ctx, &uvt->resp.u.sidr_req_resp,
- &evt->param.sidr_req_rcvd);
+ uvt->resp.u.sidr_req_resp.pkey =
+ evt->param.sidr_req_rcvd.pkey;
uvt->data_len = IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE;
break;
case IB_CM_SIDR_REP_RECEIVED:
@@ -378,31 +340,24 @@ static int ib_ucm_event_handler(struct i
struct ib_ucm_event *uevent;
struct ib_ucm_context *ctx;
int result = 0;
- int id;
ctx = cm_id->context;
- if (event->event == IB_CM_REQ_RECEIVED ||
- event->event == IB_CM_SIDR_REQ_RECEIVED)
- id = IB_UCM_CM_ID_INVALID;
- else
- id = ctx->id;
-
uevent = kmalloc(sizeof(*uevent), GFP_KERNEL);
if (!uevent)
goto err1;
memset(uevent, 0, sizeof(*uevent));
- uevent->resp.id = id;
+ uevent->ctx = ctx;
+ uevent->cm_id = cm_id;
+ uevent->resp.uid = ctx->uid;
+ uevent->resp.id = ctx->id;
uevent->resp.event = event->event;
- result = ib_ucm_event_process(ctx, event, uevent);
+ result = ib_ucm_event_process(event, uevent);
if (result)
goto err2;
- uevent->ctx = ctx;
- uevent->cm_id = (id == IB_UCM_CM_ID_INVALID) ? cm_id : NULL;
-
down(&ctx->file->mutex);
list_add_tail(&uevent->file_list, &ctx->file->events);
list_add_tail(&uevent->ctx_list, &ctx->events);
@@ -414,7 +369,7 @@ err2:
kfree(uevent);
err1:
/* Destroy new cm_id's */
- return (id == IB_UCM_CM_ID_INVALID);
+ return ib_ucm_new_cm_id(event->event);
}
static ssize_t ib_ucm_event(struct ib_ucm_file *file,
@@ -423,7 +378,7 @@ static ssize_t ib_ucm_event(struct ib_uc
{
struct ib_ucm_context *ctx;
struct ib_ucm_event_get cmd;
- struct ib_ucm_event *uevent = NULL;
+ struct ib_ucm_event *uevent;
int result = 0;
DEFINE_WAIT(wait);
@@ -436,7 +391,6 @@ static ssize_t ib_ucm_event(struct ib_uc
* wait
*/
down(&file->mutex);
-
while (list_empty(&file->events)) {
if (file->filp->f_flags & O_NONBLOCK) {
@@ -463,21 +417,18 @@ static ssize_t ib_ucm_event(struct ib_uc
uevent = list_entry(file->events.next, struct ib_ucm_event, file_list);
- if (!uevent->cm_id)
- goto user;
+ if (ib_ucm_new_cm_id(uevent->resp.event)) {
+ ctx = ib_ucm_ctx_alloc(file);
+ if (!ctx) {
+ result = -ENOMEM;
+ goto done;
+ }
- ctx = ib_ucm_ctx_alloc(file);
- if (!ctx) {
- result = -ENOMEM;
- goto done;
+ ctx->cm_id = uevent->cm_id;
+ ctx->cm_id->context = ctx;
+ uevent->resp.id = ctx->id;
}
- ctx->cm_id = uevent->cm_id;
- ctx->cm_id->context = ctx;
-
- uevent->resp.id = ctx->id;
-
-user:
if (copy_to_user((void __user *)(unsigned long)cmd.response,
&uevent->resp, sizeof(uevent->resp))) {
result = -EFAULT;
@@ -485,12 +436,10 @@ user:
}
if (uevent->data) {
-
if (cmd.data_len < uevent->data_len) {
result = -ENOMEM;
goto done;
}
-
if (copy_to_user((void __user *)(unsigned long)cmd.data,
uevent->data, uevent->data_len)) {
result = -EFAULT;
@@ -499,12 +448,10 @@ user:
}
if (uevent->info) {
-
if (cmd.info_len < uevent->info_len) {
result = -ENOMEM;
goto done;
}
-
if (copy_to_user((void __user *)(unsigned long)cmd.info,
uevent->info, uevent->info_len)) {
result = -EFAULT;
@@ -514,6 +461,7 @@ user:
list_del(&uevent->file_list);
list_del(&uevent->ctx_list);
+ uevent->ctx->events_reported++;
kfree(uevent->data);
kfree(uevent->info);
@@ -545,6 +493,7 @@ static ssize_t ib_ucm_create_id(struct i
if (!ctx)
return -ENOMEM;
+ ctx->uid = cmd.uid;
ctx->cm_id = ib_create_cm_id(ib_ucm_event_handler, ctx);
if (IS_ERR(ctx->cm_id)) {
result = PTR_ERR(ctx->cm_id);
@@ -561,7 +510,14 @@ static ssize_t ib_ucm_create_id(struct i
return 0;
err:
- ib_ucm_destroy_ctx(file, ctx->id);
+ down(&ctx_id_mutex);
+ idr_remove(&ctx_id_table, ctx->id);
+ up(&ctx_id_mutex);
+
+ if (!IS_ERR(ctx->cm_id))
+ ib_destroy_cm_id(ctx->cm_id);
+
+ kfree(ctx);
return result;
}
@@ -570,11 +526,44 @@ static ssize_t ib_ucm_destroy_id(struct
int in_len, int out_len)
{
struct ib_ucm_destroy_id cmd;
+ struct ib_ucm_destroy_id_resp resp;
+ struct ib_ucm_context *ctx;
+ int result = 0;
+
+ if (out_len < sizeof(resp))
+ return -ENOSPC;
if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
return -EFAULT;
- return ib_ucm_destroy_ctx(file, cmd.id);
+ down(&ctx_id_mutex);
+ ctx = idr_find(&ctx_id_table, cmd.id);
+ if (!ctx)
+ ctx = ERR_PTR(-ENOENT);
+ else if (ctx->file != file)
+ ctx = ERR_PTR(-EINVAL);
+ else
+ idr_remove(&ctx_id_table, ctx->id);
+ up(&ctx_id_mutex);
+
+ if (IS_ERR(ctx))
+ return PTR_ERR(ctx);
+
+ atomic_dec(&ctx->ref);
+ wait_event(ctx->wait, !atomic_read(&ctx->ref));
+
+ /* No new events will be generated after destroying the cm_id. */
+ ib_destroy_cm_id(ctx->cm_id);
+ /* Cleanup events not yet reported to the user. */
+ ib_ucm_cleanup_events(ctx);
+
+ resp.events_reported = ctx->events_reported;
+ if (copy_to_user((void __user *)(unsigned long)cmd.response,
+ &resp, sizeof(resp)))
+ result = -EFAULT;
+
+ kfree(ctx);
+ return result;
}
static ssize_t ib_ucm_attr_id(struct ib_ucm_file *file,
@@ -609,6 +598,98 @@ static ssize_t ib_ucm_attr_id(struct ib_
return result;
}
+static void ib_ucm_copy_ah_attr(struct ib_ucm_ah_attr *dest_attr,
+ struct ib_ah_attr *src_attr)
+{
+ memcpy(dest_attr->grh_dgid, src_attr->grh.dgid.raw,
+ sizeof src_attr->grh.dgid);
+ dest_attr->grh_flow_label = src_attr->grh.flow_label;
+ dest_attr->grh_sgid_index = src_attr->grh.sgid_index;
+ dest_attr->grh_hop_limit = src_attr->grh.hop_limit;
+ dest_attr->grh_traffic_class = src_attr->grh.traffic_class;
+
+ dest_attr->dlid = src_attr->dlid;
+ dest_attr->sl = src_attr->sl;
+ dest_attr->src_path_bits = src_attr->src_path_bits;
+ dest_attr->static_rate = src_attr->static_rate;
+ dest_attr->is_global = (src_attr->ah_flags & IB_AH_GRH);
+ dest_attr->port_num = src_attr->port_num;
+}
+
+static void ib_ucm_copy_qp_attr(struct ib_ucm_init_qp_attr_resp *dest_attr,
+ struct ib_qp_attr *src_attr)
+{
+ dest_attr->cur_qp_state = src_attr->cur_qp_state;
+ dest_attr->path_mtu = src_attr->path_mtu;
+ dest_attr->path_mig_state = src_attr->path_mig_state;
+ dest_attr->qkey = src_attr->qkey;
+ dest_attr->rq_psn = src_attr->rq_psn;
+ dest_attr->sq_psn = src_attr->sq_psn;
+ dest_attr->dest_qp_num = src_attr->dest_qp_num;
+ dest_attr->qp_access_flags = src_attr->qp_access_flags;
+
+ dest_attr->max_send_wr = src_attr->cap.max_send_wr;
+ dest_attr->max_recv_wr = src_attr->cap.max_recv_wr;
+ dest_attr->max_send_sge = src_attr->cap.max_send_sge;
+ dest_attr->max_recv_sge = src_attr->cap.max_recv_sge;
+ dest_attr->max_inline_data = src_attr->cap.max_inline_data;
+
+ ib_ucm_copy_ah_attr(&dest_attr->ah_attr, &src_attr->ah_attr);
+ ib_ucm_copy_ah_attr(&dest_attr->alt_ah_attr, &src_attr->alt_ah_attr);
+
+ dest_attr->pkey_index = src_attr->pkey_index;
+ dest_attr->alt_pkey_index = src_attr->alt_pkey_index;
+ dest_attr->en_sqd_async_notify = src_attr->en_sqd_async_notify;
+ dest_attr->sq_draining = src_attr->sq_draining;
+ dest_attr->max_rd_atomic = src_attr->max_rd_atomic;
+ dest_attr->max_dest_rd_atomic = src_attr->max_dest_rd_atomic;
+ dest_attr->min_rnr_timer = src_attr->min_rnr_timer;
+ dest_attr->port_num = src_attr->port_num;
+ dest_attr->timeout = src_attr->timeout;
+ dest_attr->retry_cnt = src_attr->retry_cnt;
+ dest_attr->rnr_retry = src_attr->rnr_retry;
+ dest_attr->alt_port_num = src_attr->alt_port_num;
+ dest_attr->alt_timeout = src_attr->alt_timeout;
+}
+
+static ssize_t ib_ucm_init_qp_attr(struct ib_ucm_file *file,
+ const char __user *inbuf,
+ int in_len, int out_len)
+{
+ struct ib_ucm_init_qp_attr_resp resp;
+ struct ib_ucm_init_qp_attr cmd;
+ struct ib_ucm_context *ctx;
+ struct ib_qp_attr qp_attr;
+ int result = 0;
+
+ if (out_len < sizeof(resp))
+ return -ENOSPC;
+
+ if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
+ return -EFAULT;
+
+ ctx = ib_ucm_ctx_get(file, cmd.id);
+ if (IS_ERR(ctx))
+ return PTR_ERR(ctx);
+
+ resp.qp_attr_mask = 0;
+ memset(&qp_attr, 0, sizeof qp_attr);
+ qp_attr.qp_state = cmd.qp_state;
+ result = ib_cm_init_qp_attr(ctx->cm_id, &qp_attr, &resp.qp_attr_mask);
+ if (result)
+ goto out;
+
+ ib_ucm_copy_qp_attr(&resp, &qp_attr);
+
+ if (copy_to_user((void __user *)(unsigned long)cmd.response,
+ &resp, sizeof(resp)))
+ result = -EFAULT;
+
+out:
+ ib_ucm_ctx_put(ctx);
+ return result;
+}
+
static ssize_t ib_ucm_listen(struct ib_ucm_file *file,
const char __user *inbuf,
int in_len, int out_len)
@@ -808,6 +889,7 @@ static ssize_t ib_ucm_send_rep(struct ib
ctx = ib_ucm_ctx_get(file, cmd.id);
if (!IS_ERR(ctx)) {
+ ctx->uid = cmd.uid;
result = ib_send_cm_rep(ctx->cm_id, ¶m);
ib_ucm_ctx_put(ctx);
} else
@@ -1086,6 +1168,7 @@ static ssize_t (*ucm_cmd_table[])(struct
[IB_USER_CM_CMD_SEND_SIDR_REQ] = ib_ucm_send_sidr_req,
[IB_USER_CM_CMD_SEND_SIDR_REP] = ib_ucm_send_sidr_rep,
[IB_USER_CM_CMD_EVENT] = ib_ucm_event,
+ [IB_USER_CM_CMD_INIT_QP_ATTR] = ib_ucm_init_qp_attr,
};
static ssize_t ib_ucm_write(struct file *filp, const char __user *buf,
@@ -1161,12 +1244,18 @@ static int ib_ucm_close(struct inode *in
down(&file->mutex);
while (!list_empty(&file->ctxs)) {
-
ctx = list_entry(file->ctxs.next,
struct ib_ucm_context, file_list);
-
up(&file->mutex);
- ib_ucm_destroy_ctx(file, ctx->id);
+
+ down(&ctx_id_mutex);
+ idr_remove(&ctx_id_table, ctx->id);
+ up(&ctx_id_mutex);
+
+ ib_destroy_cm_id(ctx->cm_id);
+ ib_ucm_cleanup_events(ctx);
+ kfree(ctx);
+
down(&file->mutex);
}
up(&file->mutex);
Index: linux-kernel/infiniband/core/ucm.h
===================================================================
--- linux-kernel/infiniband/core/ucm.h (revision 3109)
+++ linux-kernel/infiniband/core/ucm.h (working copy)
@@ -1,5 +1,6 @@
/*
* Copyright (c) 2005 Topspin Communications. All rights reserved.
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -43,8 +44,6 @@
#include <ib_cm.h>
#include <ib_user_cm.h>
-#define IB_UCM_CM_ID_INVALID 0xffffffff
-
struct ib_ucm_file {
struct semaphore mutex;
struct file *filp;
@@ -58,9 +57,11 @@ struct ib_ucm_context {
int id;
wait_queue_head_t wait;
atomic_t ref;
+ int events_reported;
struct ib_ucm_file *file;
struct ib_cm_id *cm_id;
+ __u64 uid;
struct list_head events; /* list of pending events. */
struct list_head file_list; /* member in file ctx list */
@@ -71,16 +72,12 @@ struct ib_ucm_event {
struct list_head file_list; /* member in file event list */
struct list_head ctx_list; /* member in ctx event list */
+ struct ib_cm_id *cm_id;
struct ib_ucm_event_resp resp;
void *data;
void *info;
int data_len;
int info_len;
- /*
- * new connection identifiers needs to be saved until
- * userspace can get a handle on them.
- */
- struct ib_cm_id *cm_id;
};
#endif /* UCM_H */
More information about the general
mailing list