[openib-general] [PATCH] librdmacm: updated librdmacm to work with proposed 2.6.20 kernel CMA
Sean Hefty
sean.hefty at intel.com
Wed Oct 25 13:49:59 PDT 2006
Updates the librdmacm to work with ABI version 3, which is the proposed
kernel changes for inclusion in 2.6.20.
Test programs are also updated.
Signed-off-by: Sean Hefty <sean.hefty at intel.com>
---
Index: include/rdma/rdma_cma_abi.h
===================================================================
--- include/rdma/rdma_cma_abi.h (revision 9192)
+++ include/rdma/rdma_cma_abi.h (working copy)
@@ -33,14 +33,15 @@
#ifndef RDMA_CMA_ABI_H
#define RDMA_CMA_ABI_H
+#include <infiniband/kern-abi.h>
#include <infiniband/sa-kern-abi.h>
/*
* This file must be kept in sync with the kernel's version of rdma_user_cm.h
*/
-#define RDMA_USER_CM_MIN_ABI_VERSION 1
-#define RDMA_USER_CM_MAX_ABI_VERSION 2
+#define RDMA_USER_CM_MIN_ABI_VERSION 3
+#define RDMA_USER_CM_MAX_ABI_VERSION 3
#define RDMA_MAX_PRIVATE_DATA 256
@@ -60,7 +61,7 @@ enum {
UCMA_CMD_GET_EVENT,
UCMA_CMD_GET_OPTION,
UCMA_CMD_SET_OPTION,
- UCMA_CMD_GET_DST_ATTR,
+ UCMA_CMD_ESTABLISH,
UCMA_CMD_JOIN_MCAST,
UCMA_CMD_LEAVE_MCAST
};
@@ -71,11 +72,6 @@ struct ucma_abi_cmd_hdr {
__u16 out;
};
-struct ucma_abi_create_id_v1 {
- __u64 uid;
- __u64 response;
-};
-
struct ucma_abi_create_id {
__u64 uid;
__u64 response;
@@ -133,7 +129,7 @@ struct ucma_abi_query_route_resp {
struct ucma_abi_conn_param {
__u32 qp_num;
- __u32 qp_type;
+ __u32 reserved;
__u8 private_data[RDMA_MAX_PRIVATE_DATA];
__u8 private_data_len;
__u8 srq;
@@ -145,6 +141,15 @@ struct ucma_abi_conn_param {
__u8 valid;
};
+struct ucma_abi_ud_param {
+ __u32 qp_num;
+ __u32 qkey;
+ struct ibv_kern_ah_attr ah_attr;
+ __u8 private_data[RDMA_MAX_PRIVATE_DATA];
+ __u8 private_data_len;
+ __u8 reserved[7];
+};
+
struct ucma_abi_connect {
struct ucma_abi_conn_param conn_param;
__u32 id;
@@ -180,25 +185,13 @@ struct ucma_abi_init_qp_attr {
__u32 qp_state;
};
-struct ucma_abi_join_mcast {
- __u32 id;
- struct sockaddr_in6 addr;
- __u64 uid;
-};
-
-struct ucma_abi_leave_mcast {
+struct ucma_abi_establish {
__u32 id;
- struct sockaddr_in6 addr;
-};
-
-struct ucma_abi_dst_attr_resp {
- __u32 remote_qpn;
- __u32 remote_qkey;
- struct ibv_kern_ah_attr ah_attr;
};
-struct ucma_abi_get_dst_attr {
- __u64 response;
+struct ucma_abi_join_mcast {
+ __u64 response; /* ucma_abi_create_id_resp */
+ __u64 uid;
struct sockaddr_in6 addr;
__u32 id;
};
@@ -212,30 +205,10 @@ struct ucma_abi_event_resp {
__u32 id;
__u32 event;
__u32 status;
- __u8 private_data_len;
- __u8 reserved[3];
- __u8 private_data[RDMA_MAX_PRIVATE_DATA];
-};
-
-struct ucma_abi_get_option {
- __u64 response;
- __u64 optval;
- __u32 id;
- __u32 level;
- __u32 optname;
- __u32 optlen;
-};
-
-struct ucma_abi_get_option_resp {
- __u32 optlen;
-};
-
-struct ucma_abi_set_option {
- __u64 optval;
- __u32 id;
- __u32 level;
- __u32 optname;
- __u32 optlen;
+ union {
+ struct ucma_abi_conn_param conn;
+ struct ucma_abi_ud_param ud;
+ } param;
};
#endif /* RDMA_CMA_ABI_H */
Index: include/rdma/rdma_cma.h
===================================================================
--- include/rdma/rdma_cma.h (revision 9272)
+++ include/rdma/rdma_cma.h (working copy)
@@ -61,11 +61,11 @@ enum rdma_port_space {
RDMA_PS_UDP = 0x0111,
};
-/* Protocol levels for get/set options. */
-enum {
- RDMA_PROTO_IP = 0,
- RDMA_PROTO_IB = 1,
-};
+/*
+ * Global qkey value for all UD QPs and multicast groups created via the
+ * RDMA CM.
+ */
+#define RDMA_UD_QKEY 0x01234567
struct ib_addr {
union ibv_gid sgid;
@@ -74,8 +74,12 @@ struct ib_addr {
};
struct rdma_addr {
- struct sockaddr_in6 src_addr;
- struct sockaddr_in6 dst_addr;
+ struct sockaddr src_addr;
+ uint8_t src_pad[sizeof(struct sockaddr_storage) -
+ sizeof(struct sockaddr)];
+ struct sockaddr dst_addr;
+ uint8_t dst_pad[sizeof(struct sockaddr_storage) -
+ sizeof(struct sockaddr)];
union {
struct ib_addr ibaddr;
} addr;
@@ -101,11 +105,25 @@ struct rdma_cm_id {
uint8_t port_num;
};
-struct rdma_multicast_data {
- void *context;
- struct sockaddr addr;
- uint8_t pad[sizeof(struct sockaddr_in6) -
- sizeof(struct sockaddr)];
+struct rdma_conn_param {
+ const void *private_data;
+ uint8_t private_data_len;
+ uint8_t responder_resources;
+ uint8_t initiator_depth;
+ uint8_t flow_control;
+ uint8_t retry_count; /* ignored when accepting */
+ uint8_t rnr_retry_count;
+ /* Fields below ignored if a QP is created on the rdma_cm_id. */
+ uint8_t srq;
+ uint32_t qp_num;
+};
+
+struct rdma_ud_param {
+ const void *private_data;
+ uint8_t private_data_len;
+ struct ibv_ah_attr ah_attr;
+ uint32_t qp_num;
+ uint32_t qkey;
};
struct rdma_cm_event {
@@ -113,8 +131,10 @@ struct rdma_cm_event {
struct rdma_cm_id *listen_id;
enum rdma_cm_event_type event;
int status;
- void *private_data;
- uint8_t private_data_len;
+ union {
+ struct rdma_conn_param conn;
+ struct rdma_ud_param ud;
+ } param;
};
/**
@@ -206,20 +226,6 @@ int rdma_create_qp(struct rdma_cm_id *id
*/
void rdma_destroy_qp(struct rdma_cm_id *id);
-struct rdma_conn_param {
- const void *private_data;
- uint8_t private_data_len;
- uint8_t responder_resources;
- uint8_t initiator_depth;
- uint8_t flow_control;
- uint8_t retry_count; /* ignored when accepting */
- uint8_t rnr_retry_count;
- /* Fields below ignored if a QP is created on the rdma_cm_id. */
- uint8_t srq;
- uint32_t qp_num;
- enum ibv_qp_type qp_type;
-};
-
/**
* rdma_connect - Initiate an active connection request.
*
@@ -251,6 +257,16 @@ int rdma_reject(struct rdma_cm_id *id, c
uint8_t private_data_len);
/**
+ * rdma_establish - Forces a connection state to established.
+ * @id: Connection identifier to transition to established.
+ *
+ * This routine should be invoked by users who receive messages on a
+ * QP before being notified that the connection has been established by the
+ * RDMA CM.
+ */
+int rdma_establish(struct rdma_cm_id *id);
+
+/**
* rdma_disconnect - This function disconnects the associated QP and
* transitions it into the error state.
*/
@@ -298,40 +314,17 @@ int rdma_get_cm_event(struct rdma_event_
*/
int rdma_ack_cm_event(struct rdma_cm_event *event);
-/**
- * rdma_get_option - Retrieve options for an rdma_cm_id.
- * @id: Communication identifier to retrieve option for.
- * @level: Protocol level of the option to retrieve.
- * @optname: Name of the option to retrieve.
- * @optval: Buffer to receive the returned options.
- * @optlen: On input, the size of the %optval buffer. On output, the
- * size of the returned data.
- */
-int rdma_get_option(struct rdma_cm_id *id, int level, int optname,
- void *optval, size_t *optlen);
-
-/**
- * rdma_set_option - Set options for an rdma_cm_id.
- * @id: Communication identifier to set option for.
- * @level: Protocol level of the option to set.
- * @optname: Name of the option to set.
- * @optval: Reference to the option data.
- * @optlen: The size of the %optval buffer.
- */
-int rdma_set_option(struct rdma_cm_id *id, int level, int optname,
- void *optval, size_t optlen);
-
static inline uint16_t rdma_get_src_port(struct rdma_cm_id *id)
{
- return id->route.addr.src_addr.sin6_family == PF_INET6 ?
- id->route.addr.src_addr.sin6_port :
+ return id->route.addr.src_addr.sa_family == PF_INET6 ?
+ ((struct sockaddr_in6 *) &id->route.addr.src_addr)->sin6_port :
((struct sockaddr_in *) &id->route.addr.src_addr)->sin_port;
}
static inline uint16_t rdma_get_dst_port(struct rdma_cm_id *id)
{
- return id->route.addr.dst_addr.sin6_family == PF_INET6 ?
- id->route.addr.dst_addr.sin6_port :
+ return id->route.addr.dst_addr.sa_family == PF_INET6 ?
+ ((struct sockaddr_in6 *) &id->route.addr.dst_addr)->sin6_port :
((struct sockaddr_in *) &id->route.addr.dst_addr)->sin_port;
}
Index: src/cma.c
===================================================================
--- src/cma.c (revision 9696)
+++ src/cma.c (working copy)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ * Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -54,7 +54,6 @@
#include <infiniband/marshall.h>
#include <rdma/rdma_cma.h>
#include <rdma/rdma_cma_abi.h>
-#include <rdma/rdma_cma_ib.h>
#define PFX "librdmacm: "
@@ -116,6 +115,28 @@ struct cma_id_private {
pthread_cond_t cond;
pthread_mutex_t mut;
uint32_t handle;
+ struct cma_multicast *mc_list;
+};
+
+struct cma_multicast {
+ struct cma_multicast *next;
+ struct cma_id_private *id_priv;
+ void *context;
+ int events_completed;
+ pthread_cond_t cond;
+ uint32_t handle;
+ union ibv_gid mgid;
+ uint16_t mlid;
+ struct sockaddr addr;
+ uint8_t pad[sizeof(struct sockaddr_in6) -
+ sizeof(struct sockaddr)];
+};
+
+struct cma_event {
+ struct rdma_cm_event event;
+ uint8_t private_data[RDMA_MAX_PRIVATE_DATA];
+ struct cma_id_private *id_priv;
+ struct cma_multicast *mc;
};
static struct cma_device *cma_dev_array;
@@ -335,41 +356,6 @@ err: ucma_free_id(id_priv);
return NULL;
}
-static int ucma_create_id_v1(struct rdma_event_channel *channel,
- struct rdma_cm_id **id, void *context,
- enum rdma_port_space ps)
-{
- struct ucma_abi_create_id_resp *resp;
- struct ucma_abi_create_id_v1 *cmd;
- struct cma_id_private *id_priv;
- void *msg;
- int ret, size;
-
- if (ps != RDMA_PS_TCP) {
- fprintf(stderr, "librdmacm: Kernel ABI does not support "
- "requested port space.\n");
- return -EPROTONOSUPPORT;
- }
-
- id_priv = ucma_alloc_id(channel, context, ps);
- if (!id_priv)
- return -ENOMEM;
-
- CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_CREATE_ID, size);
- cmd->uid = (uintptr_t) id_priv;
-
- ret = write(channel->fd, msg, size);
- if (ret != size)
- goto err;
-
- id_priv->handle = resp->id;
- *id = &id_priv->id;
- return 0;
-
-err: ucma_free_id(id_priv);
- return ret;
-}
-
int rdma_create_id(struct rdma_event_channel *channel,
struct rdma_cm_id **id, void *context,
enum rdma_port_space ps)
@@ -384,9 +370,6 @@ int rdma_create_id(struct rdma_event_cha
if (ret)
return ret;
- if (abi_ver == 1)
- return ucma_create_id_v1(channel, id, context, ps);
-
id_priv = ucma_alloc_id(channel, context, ps);
if (!id_priv)
return -ENOMEM;
@@ -492,9 +475,9 @@ static int ucma_query_route(struct rdma_
sizeof id->route.addr.addr.ibaddr.dgid);
id->route.addr.addr.ibaddr.pkey = resp->ib_route[0].pkey;
memcpy(&id->route.addr.src_addr, &resp->src_addr,
- sizeof id->route.addr.src_addr);
+ sizeof resp->src_addr);
memcpy(&id->route.addr.dst_addr, &resp->dst_addr,
- sizeof id->route.addr.dst_addr);
+ sizeof resp->dst_addr);
if (!id_priv->cma_dev && resp->node_guid) {
ret = ucma_get_device(id_priv, resp->node_guid);
@@ -696,7 +679,7 @@ static int ucma_init_ib_qp(struct cma_id
qp_attr.port_num = id_priv->id.port_num;
qp_attr.qp_state = IBV_QPS_INIT;
- qp_attr.qp_access_flags = IBV_ACCESS_LOCAL_WRITE;
+ qp_attr.qp_access_flags = 0;
return ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_ACCESS_FLAGS |
IBV_QP_PKEY_INDEX | IBV_QP_PORT);
}
@@ -767,11 +750,9 @@ void rdma_destroy_qp(struct rdma_cm_id *
static void ucma_copy_conn_param_to_kern(struct ucma_abi_conn_param *dst,
struct rdma_conn_param *src,
- uint32_t qp_num,
- enum ibv_qp_type qp_type, uint8_t srq)
+ uint32_t qp_num, uint8_t srq)
{
dst->qp_num = qp_num;
- dst->qp_type = qp_type;
dst->srq = srq;
dst->responder_resources = src->responder_resources;
dst->initiator_depth = src->initiator_depth;
@@ -799,12 +780,11 @@ int rdma_connect(struct rdma_cm_id *id,
cmd->id = id_priv->handle;
if (id->qp)
ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
- id->qp->qp_num, id->qp->qp_type,
+ id->qp->qp_num,
(id->qp->srq != NULL));
else
ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
conn_param->qp_num,
- conn_param->qp_type,
conn_param->srq);
ret = write(id->channel->fd, msg, size);
@@ -852,12 +832,11 @@ int rdma_accept(struct rdma_cm_id *id, s
cmd->uid = (uintptr_t) id_priv;
if (id->qp)
ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
- id->qp->qp_num, id->qp->qp_type,
+ id->qp->qp_num,
(id->qp->srq != NULL));
else
ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
conn_param->qp_num,
- conn_param->qp_type,
conn_param->srq);
ret = write(id->channel->fd, msg, size);
@@ -894,6 +873,24 @@ int rdma_reject(struct rdma_cm_id *id, c
return 0;
}
+int rdma_establish(struct rdma_cm_id *id)
+{
+ struct ucma_abi_establish *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_ESTABLISH, size);
+
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ ret = write(id->channel->fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ return 0;
+}
+
int rdma_disconnect(struct rdma_cm_id *id)
{
struct ucma_abi_disconnect *cmd;
@@ -929,74 +926,102 @@ int rdma_join_multicast(struct rdma_cm_i
void *context)
{
struct ucma_abi_join_mcast *cmd;
+ struct ucma_abi_create_id_resp *resp;
struct cma_id_private *id_priv;
+ struct cma_multicast *mc, **pos;
void *msg;
int ret, size, addrlen;
+ id_priv = container_of(id, struct cma_id_private, id);
addrlen = ucma_addrlen(addr);
if (!addrlen)
return -EINVAL;
- CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_JOIN_MCAST, size);
- id_priv = container_of(id, struct cma_id_private, id);
+ mc = malloc(sizeof *mc);
+ if (!mc)
+ return -ENOMEM;
+
+ memset(mc, 0, sizeof *mc);
+ mc->context = context;
+ mc->id_priv = id_priv;
+ memcpy(&mc->addr, addr, addrlen);
+ if (pthread_cond_init(&id_priv->cond, NULL)) {
+ ret = -1;
+ goto err1;
+ }
+
+ pthread_mutex_lock(&id_priv->mut);
+ mc->next = id_priv->mc_list;
+ id_priv->mc_list = mc;
+ pthread_mutex_unlock(&id_priv->mut);
+
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_JOIN_MCAST, size);
cmd->id = id_priv->handle;
memcpy(&cmd->addr, addr, addrlen);
- cmd->uid = (uintptr_t) context;
+ cmd->uid = (uintptr_t) mc;
ret = write(id->channel->fd, msg, size);
- if (ret != size)
- return (ret > 0) ? -ENODATA : ret;
+ if (ret != size) {
+ ret = (ret > 0) ? -ENODATA : ret;
+ goto err2;
+ }
+ mc->handle = resp->id;
return 0;
+err2:
+ pthread_mutex_lock(&id_priv->mut);
+ for (pos = &id_priv->mc_list; *pos != mc; pos = &(*pos)->next)
+ ;
+ *pos = mc->next;
+ pthread_mutex_unlock(&id_priv->mut);
+err1:
+ free(mc);
+ return ret;
}
int rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
{
- struct ucma_abi_leave_mcast *cmd;
+ struct ucma_abi_destroy_id *cmd;
+ struct ucma_abi_destroy_id_resp *resp;
struct cma_id_private *id_priv;
+ struct cma_multicast *mc, **pos;
void *msg;
int ret, size, addrlen;
- struct ibv_ah_attr ah_attr;
- uint32_t qp_info;
addrlen = ucma_addrlen(addr);
if (!addrlen)
return -EINVAL;
- CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_LEAVE_MCAST, size);
id_priv = container_of(id, struct cma_id_private, id);
- cmd->id = id_priv->handle;
- memcpy(&cmd->addr, addr, addrlen);
+ pthread_mutex_lock(&id_priv->mut);
+ for (pos = &id_priv->mc_list; *pos; pos = &(*pos)->next)
+ if (!memcmp(&(*pos)->addr, addr, addrlen))
+ break;
- if (id->qp) {
- ret = rdma_get_dst_attr(id, addr, &ah_attr, &qp_info, &qp_info);
- if (ret)
- goto out;
+ mc = *pos;
+ if (*pos)
+ *pos = mc->next;
+ pthread_mutex_unlock(&id_priv->mut);
+ if (!mc)
+ return -EADDRNOTAVAIL;
- ret = ibv_detach_mcast(id->qp, &ah_attr.grh.dgid, ah_attr.dlid);
- if (ret)
- goto out;
- }
+ if (id->qp)
+ ibv_detach_mcast(id->qp, &mc->mgid, mc->mlid);
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_LEAVE_MCAST, size);
+ cmd->id = mc->handle;
+
ret = write(id->channel->fd, msg, size);
if (ret != size)
ret = (ret > 0) ? -ENODATA : ret;
-out:
- return ret;
-}
-static void ucma_copy_event_from_kern(struct rdma_cm_event *dst,
- struct ucma_abi_event_resp *src)
-{
- dst->event = src->event;
- dst->status = src->status;
- dst->private_data_len = src->private_data_len;
- if (src->private_data_len) {
- dst->private_data = dst + 1;
- memcpy(dst->private_data, src->private_data,
- src->private_data_len);
- } else
- dst->private_data = NULL;
+ pthread_mutex_lock(&id_priv->mut);
+ while (mc->events_completed < resp->events_reported)
+ pthread_cond_wait(&mc->cond, &id_priv->mut);
+ pthread_mutex_unlock(&id_priv->mut);
+
+ free(mc);
+ return ret;
}
static void ucma_complete_event(struct cma_id_private *id_priv)
@@ -1007,38 +1032,49 @@ static void ucma_complete_event(struct c
pthread_mutex_unlock(&id_priv->mut);
}
+static void ucma_complete_mc_event(struct cma_multicast *mc)
+{
+ pthread_mutex_lock(&mc->id_priv->mut);
+ mc->events_completed++;
+ pthread_cond_signal(&mc->cond);
+ mc->id_priv->events_completed++;
+ pthread_cond_signal(&mc->id_priv->cond);
+ pthread_mutex_unlock(&mc->id_priv->mut);
+}
+
int rdma_ack_cm_event(struct rdma_cm_event *event)
{
- struct rdma_cm_id *id;
+ struct cma_event *evt;
if (!event)
return -EINVAL;
- id = (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) ?
- event->listen_id : event->id;
+ evt = container_of(event, struct cma_event, event);
- ucma_complete_event(container_of(id, struct cma_id_private, id));
- free(event);
+ if (evt->mc)
+ ucma_complete_mc_event(evt->mc);
+ else
+ ucma_complete_event(evt->id_priv);
+ free(evt);
return 0;
}
-static int ucma_process_conn_req(struct rdma_cm_event *event,
+static int ucma_process_conn_req(struct cma_event *evt,
uint32_t handle)
{
- struct cma_id_private *listen_id_priv, *id_priv;
+ struct cma_id_private *id_priv;
int ret;
- listen_id_priv = container_of(event->id, struct cma_id_private, id);
- id_priv = ucma_alloc_id(event->id->channel, event->id->context,
- event->id->ps);
+ id_priv = ucma_alloc_id(evt->id_priv->id.channel,
+ evt->id_priv->id.context, evt->id_priv->id.ps);
if (!id_priv) {
- ucma_destroy_kern_id(event->id->channel->fd, handle);
+ ucma_destroy_kern_id(evt->id_priv->id.channel->fd, handle);
ret = -ENOMEM;
goto err;
}
- event->listen_id = event->id;
- event->id = &id_priv->id;
+ evt->event.listen_id = &evt->id_priv->id;
+ evt->event.id = &id_priv->id;
id_priv->handle = handle;
ret = ucma_query_route(&id_priv->id);
@@ -1049,7 +1085,7 @@ static int ucma_process_conn_req(struct
return 0;
err:
- ucma_complete_event(listen_id_priv);
+ ucma_complete_event(evt->id_priv);
return ret;
}
@@ -1093,34 +1129,54 @@ static int ucma_process_establish(struct
return ret;
}
-static void ucma_process_mcast(struct rdma_cm_id *id, struct rdma_cm_event *evt)
+static int ucma_process_join(struct cma_event *evt)
{
- struct ucma_abi_join_mcast kmc_data;
- struct rdma_multicast_data *mc_data;
- struct ibv_ah_attr ah_attr;
- uint32_t qp_info;
-
- kmc_data = *(struct ucma_abi_join_mcast *) evt->private_data;
-
- mc_data = evt->private_data;
- mc_data->context = (void *) (uintptr_t) kmc_data.uid;
- memcpy(&mc_data->addr, &kmc_data.addr,
- ucma_addrlen((struct sockaddr *) &kmc_data.addr));
-
- if (evt->status || !id->qp)
- return;
-
- evt->status = rdma_get_dst_attr(id, &mc_data->addr, &ah_attr,
- &qp_info, &qp_info);
- if (evt->status)
- goto err;
+ evt->mc->mgid = evt->event.param.ud.ah_attr.grh.dgid;
+ evt->mc->mlid = evt->event.param.ud.ah_attr.dlid;
- evt->status = ibv_attach_mcast(id->qp, &ah_attr.grh.dgid, ah_attr.dlid);
- if (evt->status)
- goto err;
- return;
-err:
- evt->event = RDMA_CM_EVENT_MULTICAST_ERROR;
+ if (evt->id_priv->id.qp)
+ return ibv_attach_mcast(evt->id_priv->id.qp,
+ &evt->mc->mgid, evt->mc->mlid);
+ else
+ return 0;
+}
+
+static void ucma_copy_conn_event(struct cma_event *event,
+ struct ucma_abi_conn_param *src)
+{
+ struct rdma_conn_param *dst = &event->event.param.conn;
+
+ dst->private_data_len = src->private_data_len;
+ if (src->private_data_len) {
+ dst->private_data = &event->private_data;
+ memcpy(&event->private_data, src->private_data,
+ src->private_data_len);
+ }
+
+ dst->responder_resources = src->responder_resources;
+ dst->initiator_depth = src->initiator_depth;
+ dst->flow_control = src->flow_control;
+ dst->retry_count = src->retry_count;
+ dst->rnr_retry_count = src->rnr_retry_count;
+ dst->srq = src->srq;
+ dst->qp_num = src->qp_num;
+}
+
+static void ucma_copy_ud_event(struct cma_event *event,
+ struct ucma_abi_ud_param *src)
+{
+ struct rdma_ud_param *dst = &event->event.param.ud;
+
+ dst->private_data_len = src->private_data_len;
+ if (src->private_data_len) {
+ dst->private_data = &event->private_data;
+ memcpy(&event->private_data, src->private_data,
+ src->private_data_len);
+ }
+
+ ibv_copy_ah_attr_from_kern(&dst->ah_attr, &src->ah_attr);
+ dst->qp_num = src->qp_num;
+ dst->qkey = src->qkey;
}
int rdma_get_cm_event(struct rdma_event_channel *channel,
@@ -1128,8 +1184,7 @@ int rdma_get_cm_event(struct rdma_event_
{
struct ucma_abi_event_resp *resp;
struct ucma_abi_get_event *cmd;
- struct cma_id_private *id_priv;
- struct rdma_cm_event *evt;
+ struct cma_event *evt;
void *msg;
int ret, size;
@@ -1140,155 +1195,119 @@ int rdma_get_cm_event(struct rdma_event_
if (!event)
return -EINVAL;
- evt = malloc(sizeof *evt + RDMA_MAX_PRIVATE_DATA);
+ evt = malloc(sizeof *evt);
if (!evt)
return -ENOMEM;
retry:
+ memset(evt, 0, sizeof *evt);
CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_GET_EVENT, size);
ret = write(channel->fd, msg, size);
if (ret != size) {
free(evt);
return (ret > 0) ? -ENODATA : ret;
}
-
- id_priv = (void *) (uintptr_t) resp->uid;
- evt->id = &id_priv->id;
- ucma_copy_event_from_kern(evt, resp);
- switch (evt->event) {
+ evt->event.event = resp->event;
+ switch (resp->event) {
case RDMA_CM_EVENT_ADDR_RESOLVED:
- evt->status = ucma_query_route(&id_priv->id);
- if (evt->status)
- evt->event = RDMA_CM_EVENT_ADDR_ERROR;
+ evt->id_priv = (void *) (uintptr_t) resp->uid;
+ evt->event.id = &evt->id_priv->id;
+ evt->event.status = ucma_query_route(&evt->id_priv->id);
+ if (evt->event.status)
+ evt->event.event = RDMA_CM_EVENT_ADDR_ERROR;
break;
case RDMA_CM_EVENT_ROUTE_RESOLVED:
- evt->status = ucma_query_route(&id_priv->id);
- if (evt->status)
- evt->event = RDMA_CM_EVENT_ROUTE_ERROR;
+ evt->id_priv = (void *) (uintptr_t) resp->uid;
+ evt->event.id = &evt->id_priv->id;
+ evt->event.status = ucma_query_route(&evt->id_priv->id);
+ if (evt->event.status)
+ evt->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
break;
case RDMA_CM_EVENT_CONNECT_REQUEST:
+ evt->id_priv = (void *) (uintptr_t) resp->uid;
+ if (evt->id_priv->id.ps == RDMA_PS_TCP)
+ ucma_copy_conn_event(evt, &resp->param.conn);
+ else
+ ucma_copy_ud_event(evt, &resp->param.ud);
+
ret = ucma_process_conn_req(evt, resp->id);
if (ret)
goto retry;
break;
case RDMA_CM_EVENT_CONNECT_RESPONSE:
- evt->status = ucma_process_conn_resp(id_priv);
- if (!evt->status)
- evt->event = RDMA_CM_EVENT_ESTABLISHED;
+ evt->id_priv = (void *) (uintptr_t) resp->uid;
+ evt->event.id = &evt->id_priv->id;
+ ucma_copy_conn_event(evt, &resp->param.conn);
+ evt->event.status = ucma_process_conn_resp(evt->id_priv);
+ if (!evt->event.status)
+ evt->event.event = RDMA_CM_EVENT_ESTABLISHED;
else {
- evt->event = RDMA_CM_EVENT_CONNECT_ERROR;
- id_priv->connect_error = 1;
+ evt->event.event = RDMA_CM_EVENT_CONNECT_ERROR;
+ evt->id_priv->connect_error = 1;
}
break;
case RDMA_CM_EVENT_ESTABLISHED:
- if (id_priv->id.ps == RDMA_PS_UDP)
+ evt->id_priv = (void *) (uintptr_t) resp->uid;
+ evt->event.id = &evt->id_priv->id;
+ if (evt->id_priv->id.ps == RDMA_PS_UDP) {
+ ucma_copy_ud_event(evt, &resp->param.ud);
break;
+ }
- evt->status = ucma_process_establish(&id_priv->id);
- if (evt->status) {
- evt->event = RDMA_CM_EVENT_CONNECT_ERROR;
- id_priv->connect_error = 1;
+ ucma_copy_conn_event(evt, &resp->param.conn);
+ evt->event.status = ucma_process_establish(&evt->id_priv->id);
+ if (evt->event.status) {
+ evt->event.event = RDMA_CM_EVENT_CONNECT_ERROR;
+ evt->id_priv->connect_error = 1;
}
break;
case RDMA_CM_EVENT_REJECTED:
- if (id_priv->connect_error) {
- ucma_complete_event(id_priv);
+ evt->id_priv = (void *) (uintptr_t) resp->uid;
+ if (evt->id_priv->connect_error) {
+ ucma_complete_event(evt->id_priv);
goto retry;
}
- ucma_modify_qp_err(evt->id);
+ evt->event.id = &evt->id_priv->id;
+ ucma_copy_conn_event(evt, &resp->param.conn);
+ ucma_modify_qp_err(evt->event.id);
break;
case RDMA_CM_EVENT_DISCONNECTED:
- if (id_priv->connect_error) {
- ucma_complete_event(id_priv);
+ evt->id_priv = (void *) (uintptr_t) resp->uid;
+ if (evt->id_priv->connect_error) {
+ ucma_complete_event(evt->id_priv);
goto retry;
}
+ evt->event.id = &evt->id_priv->id;
+ ucma_copy_conn_event(evt, &resp->param.conn);
break;
case RDMA_CM_EVENT_MULTICAST_JOIN:
+ evt->mc = (void *) (uintptr_t) resp->uid;
+ evt->id_priv = evt->mc->id_priv;
+ evt->event.id = &evt->id_priv->id;
+ ucma_copy_ud_event(evt, &resp->param.ud);
+ evt->event.param.ud.private_data = evt->mc->context;
+ evt->event.status = ucma_process_join(evt);
+ if (evt->event.status)
+ evt->event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
+ break;
case RDMA_CM_EVENT_MULTICAST_ERROR:
- ucma_process_mcast(&id_priv->id, evt);
+ evt->mc = (void *) (uintptr_t) resp->uid;
+ evt->id_priv = evt->mc->id_priv;
+ evt->event.id = &evt->id_priv->id;
+ evt->event.status = resp->status;
+ evt->event.param.ud.private_data = evt->mc->context;
break;
default:
+ evt->id_priv = (void *) (uintptr_t) resp->uid;
+ evt->event.id = &evt->id_priv->id;
+ if (evt->id_priv->id.ps == RDMA_PS_TCP)
+ ucma_copy_conn_event(evt, &resp->param.conn);
+ else
+ ucma_copy_ud_event(evt, &resp->param.ud);
break;
}
- *event = evt;
- return 0;
-}
-
-int rdma_get_option(struct rdma_cm_id *id, int level, int optname,
- void *optval, size_t *optlen)
-{
- struct ucma_abi_get_option_resp *resp;
- struct ucma_abi_get_option *cmd;
- struct cma_id_private *id_priv;
- void *msg;
- int ret, size;
-
- CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_GET_OPTION, size);
- id_priv = container_of(id, struct cma_id_private, id);
- cmd->id = id_priv->handle;
- cmd->optval = (uintptr_t) optval;
- cmd->level = level;
- cmd->optname = optname;
- cmd->optlen = *optlen;
-
- ret = write(id->channel->fd, msg, size);
- if (ret != size)
- return (ret > 0) ? -ENODATA : ret;
-
- *optlen = resp->optlen;
- return 0;
-}
-
-int rdma_set_option(struct rdma_cm_id *id, int level, int optname,
- void *optval, size_t optlen)
-{
- struct ucma_abi_set_option *cmd;
- struct cma_id_private *id_priv;
- void *msg;
- int ret, size;
-
- CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_SET_OPTION, size);
- id_priv = container_of(id, struct cma_id_private, id);
- cmd->id = id_priv->handle;
- cmd->optval = (uintptr_t) optval;
- cmd->level = level;
- cmd->optname = optname;
- cmd->optlen = optlen;
-
- ret = write(id->channel->fd, msg, size);
- if (ret != size)
- return (ret > 0) ? -ENODATA : ret;
-
- return 0;
-}
-
-int rdma_get_dst_attr(struct rdma_cm_id *id, struct sockaddr *addr,
- struct ibv_ah_attr *ah_attr, uint32_t *remote_qpn,
- uint32_t *remote_qkey)
-{
- struct ucma_abi_dst_attr_resp *resp;
- struct ucma_abi_get_dst_attr *cmd;
- struct cma_id_private *id_priv;
- void *msg;
- int ret, size, addrlen;
-
- addrlen = ucma_addrlen(addr);
- if (!addrlen)
- return -EINVAL;
-
- CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_GET_DST_ATTR, size);
- id_priv = container_of(id, struct cma_id_private, id);
- cmd->id = id_priv->handle;
- memcpy(&cmd->addr, addr, addrlen);
-
- ret = write(id->channel->fd, msg, size);
- if (ret != size)
- return (ret > 0) ? -ENODATA : ret;
-
- ibv_copy_ah_attr_from_kern(ah_attr, &resp->ah_attr);
- *remote_qpn = resp->remote_qpn;
- *remote_qkey = resp->remote_qkey;
+ *event = &evt->event;
return 0;
}
Index: Makefile.am
===================================================================
--- Makefile.am (revision 9192)
+++ Makefile.am (working copy)
@@ -31,12 +31,10 @@ examples_mckey_LDADD = $(top_builddir)/s
librdmacmincludedir = $(includedir)/rdma
librdmacminclude_HEADERS = include/rdma/rdma_cma_abi.h \
- include/rdma/rdma_cma.h \
- include/rdma/rdma_cma_ib.h
+ include/rdma/rdma_cma.h
EXTRA_DIST = include/rdma/rdma_cma_abi.h \
include/rdma/rdma_cma.h \
- include/rdma/rdma_cma_ib.h \
src/librdmacm.map \
librdmacm.spec.in
Index: examples/mckey.c
===================================================================
--- examples/mckey.c (revision 9208)
+++ examples/mckey.c (working copy)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ * Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -42,9 +42,9 @@
#include <netdb.h>
#include <byteswap.h>
#include <unistd.h>
+#include <getopt.h>
#include <rdma/rdma_cma.h>
-#include <rdma/rdma_cma_ib.h>
struct cmatest_node {
int id;
@@ -76,6 +76,8 @@ static int connections = 1;
static int message_size = 100;
static int message_count = 10;
static int is_sender;
+static char *dst_addr;
+static char *src_addr;
static int create_message(struct cmatest_node *node)
{
@@ -239,19 +241,12 @@ err:
return ret;
}
-static int join_handler(struct cmatest_node *node)
+static int join_handler(struct cmatest_node *node,
+ struct rdma_ud_param *param)
{
- struct ibv_ah_attr ah_attr;
- int ret;
-
- ret = rdma_get_dst_attr(node->cma_id, test.dst_addr, &ah_attr,
- &node->remote_qpn, &node->remote_qkey);
- if (ret) {
- printf("mckey: failure getting destination attributes\n");
- goto err;
- }
-
- node->ah = ibv_create_ah(node->pd, &ah_attr);
+ node->remote_qpn = param->qp_num;
+ node->remote_qkey = param->qkey;
+ node->ah = ibv_create_ah(node->pd, ¶m->ah_attr);
if (!node->ah) {
printf("mckey: failure creating address handle\n");
goto err;
@@ -262,7 +257,7 @@ static int join_handler(struct cmatest_n
return 0;
err:
connect_error();
- return ret;
+ return -1;
}
static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
@@ -274,7 +269,7 @@ static int cma_handler(struct rdma_cm_id
ret = addr_handler(cma_id->context);
break;
case RDMA_CM_EVENT_MULTICAST_JOIN:
- ret = join_handler(cma_id->context);
+ ret = join_handler(cma_id->context, &event->param.ud);
break;
case RDMA_CM_EVENT_ADDR_ERROR:
case RDMA_CM_EVENT_ROUTE_ERROR:
@@ -411,18 +406,21 @@ out:
return ret;
}
-static int run(char *dst, char *src)
+static int run(void)
{
int i, ret;
- printf("mckey: starting client\n");
- if (src) {
- ret = get_addr(src, &test.src_in);
+ if (is_sender)
+ printf("mckey: starting client\n");
+ else
+ printf("mckey: starting server\n");
+ if (src_addr) {
+ ret = get_addr(src_addr, &test.src_in);
if (ret)
return ret;
}
- ret = get_addr(dst, &test.dst_in);
+ ret = get_addr(dst_addr, &test.dst_in);
if (ret)
return ret;
@@ -431,7 +429,7 @@ static int run(char *dst, char *src)
printf("mckey: joining\n");
for (i = 0; i < connections; i++) {
ret = rdma_resolve_addr(test.nodes[i].cma_id,
- src ? test.src_addr : NULL,
+ src_addr ? test.src_addr : NULL,
test.dst_addr, 2000);
if (ret) {
printf("mckey: failure getting addr: %d\n", ret);
@@ -472,14 +470,39 @@ out:
int main(int argc, char **argv)
{
- int ret;
+ int op, ret;
- if (argc < 3 || argc > 4) {
- printf("usage: %s {s[end] | r[ecv]} mcast_addr [bind_addr]]\n",
- argv[0]);
- exit(1);
+ while ((op = getopt(argc, argv, "m:sb:c:C:S:")) != -1) {
+ switch (op) {
+ case 'm':
+ dst_addr = optarg;
+ break;
+ case 's':
+ is_sender = 1;
+ break;
+ case 'b':
+ src_addr = optarg;
+ break;
+ case 'c':
+ connections = atoi(optarg);
+ break;
+ case 'C':
+ message_count = atoi(optarg);
+ break;
+ case 'S':
+ message_size = atoi(optarg);
+ break;
+ default:
+ printf("usage: %s\n", argv[0]);
+ printf("\t-m multicast_address\n");
+ printf("\t[-s(ender)]\n");
+ printf("\t[-b bind_address]\n");
+ printf("\t[-c connections]\n");
+ printf("\t[-C message_count]\n");
+ printf("\t[-S message_size]\n");
+ exit(1);
+ }
}
- is_sender = (argv[1][0] == 's');
test.dst_addr = (struct sockaddr *) &test.dst_in;
test.src_addr = (struct sockaddr *) &test.src_in;
@@ -494,7 +517,7 @@ int main(int argc, char **argv)
if (alloc_nodes())
exit(1);
- ret = run(argv[2], (argc == 4) ? argv[3] : NULL);
+ ret = run();
printf("test complete\n");
destroy_nodes();
Index: examples/udaddy.c
===================================================================
--- examples/udaddy.c (revision 9208)
+++ examples/udaddy.c (working copy)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ * Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -41,15 +41,9 @@
#include <sys/socket.h>
#include <netdb.h>
#include <byteswap.h>
+#include <getopt.h>
#include <rdma/rdma_cma.h>
-#include <rdma/rdma_cma_ib.h>
-
-/*
- * To execute:
- * Server: udaddy
- * Client: udaddy [server_addr [src_addr]]
- */
struct cmatest_node {
int id;
@@ -80,7 +74,8 @@ static struct cmatest test;
static int connections = 1;
static int message_size = 100;
static int message_count = 10;
-static int is_server;
+static char *dst_addr;
+static char *src_addr;
static int create_message(struct cmatest_node *node)
{
@@ -246,7 +241,6 @@ static int route_handler(struct cmatest_
memset(&conn_param, 0, sizeof conn_param);
conn_param.qp_num = node->cma_id->qp->qp_num;
- conn_param.qp_type = node->cma_id->qp->qp_type;
conn_param.retry_count = 5;
ret = rdma_connect(node->cma_id, &conn_param);
if (ret) {
@@ -284,7 +278,6 @@ static int connect_handler(struct rdma_c
memset(&conn_param, 0, sizeof conn_param);
conn_param.qp_num = node->cma_id->qp->qp_num;
- conn_param.qp_type = node->cma_id->qp->qp_type;
ret = rdma_accept(node->cma_id, &conn_param);
if (ret) {
printf("udaddy: failure accepting: %d\n", ret);
@@ -303,19 +296,12 @@ err1:
return ret;
}
-static int resolved_handler(struct cmatest_node *node)
+static int resolved_handler(struct cmatest_node *node,
+ struct rdma_cm_event *event)
{
- struct ibv_ah_attr ah_attr;
- int ret;
-
- ret = rdma_get_dst_attr(node->cma_id, test.dst_addr, &ah_attr,
- &node->remote_qpn, &node->remote_qkey);
- if (ret) {
- printf("udaddy: failure getting destination attributes\n");
- goto err;
- }
-
- node->ah = ibv_create_ah(node->pd, &ah_attr);
+ node->remote_qpn = event->param.ud.qp_num;
+ node->remote_qkey = event->param.ud.qkey;
+ node->ah = ibv_create_ah(node->pd, &event->param.ud.ah_attr);
if (!node->ah) {
printf("udaddy: failure creating address handle\n");
goto err;
@@ -326,7 +312,7 @@ static int resolved_handler(struct cmate
return 0;
err:
connect_error();
- return ret;
+ return -1;
}
static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
@@ -344,7 +330,7 @@ static int cma_handler(struct rdma_cm_id
ret = connect_handler(cma_id);
break;
case RDMA_CM_EVENT_ESTABLISHED:
- ret = resolved_handler(cma_id->context);
+ ret = resolved_handler(cma_id->context, event);
break;
case RDMA_CM_EVENT_ADDR_ERROR:
case RDMA_CM_EVENT_ROUTE_ERROR:
@@ -404,7 +390,7 @@ static int alloc_nodes(void)
for (i = 0; i < connections; i++) {
test.nodes[i].id = i;
- if (!is_server) {
+ if (dst_addr) {
ret = rdma_create_id(test.channel,
&test.nodes[i].cma_id,
&test.nodes[i], RDMA_PS_UDP);
@@ -475,6 +461,28 @@ static int connect_events(void)
return ret;
}
+static int get_addr(char *dst, struct sockaddr_in *addr)
+{
+ struct addrinfo *res;
+ int ret;
+
+ ret = getaddrinfo(dst, NULL, NULL, &res);
+ if (ret) {
+ printf("getaddrinfo failed - invalid hostname or IP address\n");
+ return ret;
+ }
+
+ if (res->ai_family != PF_INET) {
+ ret = -1;
+ goto out;
+ }
+
+ *addr = *(struct sockaddr_in *) res->ai_addr;
+out:
+ freeaddrinfo(res);
+ return ret;
+}
+
static int run_server(void)
{
struct rdma_cm_id *listen_id;
@@ -487,7 +495,13 @@ static int run_server(void)
return ret;
}
- test.src_in.sin_family = PF_INET;
+ if (src_addr) {
+ ret = get_addr(src_addr, &test.src_in);
+ if (ret)
+ goto out;
+ } else
+ test.src_in.sin_family = PF_INET;
+
test.src_in.sin_port = 7174;
ret = rdma_bind_addr(listen_id, test.src_addr);
if (ret) {
@@ -526,40 +540,18 @@ out:
return ret;
}
-static int get_addr(char *dst, struct sockaddr_in *addr)
-{
- struct addrinfo *res;
- int ret;
-
- ret = getaddrinfo(dst, NULL, NULL, &res);
- if (ret) {
- printf("getaddrinfo failed - invalid hostname or IP address\n");
- return ret;
- }
-
- if (res->ai_family != PF_INET) {
- ret = -1;
- goto out;
- }
-
- *addr = *(struct sockaddr_in *) res->ai_addr;
-out:
- freeaddrinfo(res);
- return ret;
-}
-
-static int run_client(char *dst, char *src)
+static int run_client(void)
{
int i, ret;
printf("udaddy: starting client\n");
- if (src) {
- ret = get_addr(src, &test.src_in);
+ if (src_addr) {
+ ret = get_addr(src_addr, &test.src_in);
if (ret)
return ret;
}
- ret = get_addr(dst, &test.dst_in);
+ ret = get_addr(dst_addr, &test.dst_in);
if (ret)
return ret;
@@ -568,7 +560,7 @@ static int run_client(char *dst, char *s
printf("udaddy: connecting\n");
for (i = 0; i < connections; i++) {
ret = rdma_resolve_addr(test.nodes[i].cma_id,
- src ? test.src_addr : NULL,
+ src_addr ? test.src_addr : NULL,
test.dst_addr, 2000);
if (ret) {
printf("udaddy: failure getting addr: %d\n", ret);
@@ -601,13 +593,35 @@ out:
int main(int argc, char **argv)
{
- int ret;
+ int op, ret;
- if (argc > 3) {
- printf("usage: %s [server_addr [src_addr]]\n", argv[0]);
- exit(1);
+ while ((op = getopt(argc, argv, "s:b:c:C:S:")) != -1) {
+ switch (op) {
+ case 's':
+ dst_addr = optarg;
+ break;
+ case 'b':
+ src_addr = optarg;
+ break;
+ case 'c':
+ connections = atoi(optarg);
+ break;
+ case 'C':
+ message_count = atoi(optarg);
+ break;
+ case 'S':
+ message_size = atoi(optarg);
+ break;
+ default:
+ printf("usage: %s\n", argv[0]);
+ printf("\t[-s server_address]\n");
+ printf("\t[-b bind_address]\n");
+ printf("\t[-c connections]\n");
+ printf("\t[-C message_count]\n");
+ printf("\t[-S message_size]\n");
+ exit(1);
+ }
}
- is_server = (argc == 1);
test.dst_addr = (struct sockaddr *) &test.dst_in;
test.src_addr = (struct sockaddr *) &test.src_in;
@@ -622,10 +636,10 @@ int main(int argc, char **argv)
if (alloc_nodes())
exit(1);
- if (is_server)
- ret = run_server();
+ if (dst_addr)
+ ret = run_client();
else
- ret = run_client(argv[1], (argc == 3) ? argv[2] : NULL);
+ ret = run_server();
printf("test complete\n");
destroy_nodes();
Index: examples/cmatose.c
===================================================================
--- examples/cmatose.c (revision 9192)
+++ examples/cmatose.c (working copy)
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ * Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -41,6 +41,7 @@
#include <sys/socket.h>
#include <netdb.h>
#include <byteswap.h>
+#include <getopt.h>
#include <rdma/rdma_cma.h>
@@ -52,12 +53,6 @@ static inline uint64_t cpu_to_be64(uint6
static inline uint32_t cpu_to_be32(uint32_t x) { return bswap_32(x); }
#endif
-/*
- * To execute:
- * Server: rdma_cmatose
- * Client: rdma_cmatose <dst_ip>
- */
-
struct cmatest_node {
int id;
struct rdma_cm_id *cma_id;
@@ -85,7 +80,8 @@ static struct cmatest test;
static int connections = 1;
static int message_size = 100;
static int message_count = 10;
-static int is_server;
+static char *dst_addr;
+static char *src_addr;
static int create_message(struct cmatest_node *node)
{
@@ -377,7 +373,7 @@ static int alloc_nodes(void)
for (i = 0; i < connections; i++) {
test.nodes[i].id = i;
- if (!is_server) {
+ if (dst_addr) {
ret = rdma_create_id(test.channel,
&test.nodes[i].cma_id,
&test.nodes[i], RDMA_PS_TCP);
@@ -460,6 +456,28 @@ static int disconnect_events(void)
return ret;
}
+static int get_addr(char *dst, struct sockaddr_in *addr)
+{
+ struct addrinfo *res;
+ int ret;
+
+ ret = getaddrinfo(dst, NULL, NULL, &res);
+ if (ret) {
+ printf("getaddrinfo failed - invalid hostname or IP address\n");
+ return ret;
+ }
+
+ if (res->ai_family != PF_INET) {
+ ret = -1;
+ goto out;
+ }
+
+ *addr = *(struct sockaddr_in *) res->ai_addr;
+out:
+ freeaddrinfo(res);
+ return ret;
+}
+
static int run_server(void)
{
struct rdma_cm_id *listen_id;
@@ -472,12 +490,18 @@ static int run_server(void)
return ret;
}
- test.src_in.sin_family = PF_INET;
+ if (src_addr) {
+ ret = get_addr(src_addr, &test.src_in);
+ if (ret)
+ goto out;
+ } else
+ test.src_in.sin_family = PF_INET;
+
test.src_in.sin_port = 7471;
ret = rdma_bind_addr(listen_id, test.src_addr);
if (ret) {
printf("cmatose: bind address failed: %d\n", ret);
- return ret;
+ goto out;
}
ret = rdma_listen(listen_id, 0);
@@ -528,40 +552,18 @@ out:
return ret;
}
-static int get_addr(char *dst, struct sockaddr_in *addr)
-{
- struct addrinfo *res;
- int ret;
-
- ret = getaddrinfo(dst, NULL, NULL, &res);
- if (ret) {
- printf("getaddrinfo failed - invalid hostname or IP address\n");
- return ret;
- }
-
- if (res->ai_family != PF_INET) {
- ret = -1;
- goto out;
- }
-
- *addr = *(struct sockaddr_in *) res->ai_addr;
-out:
- freeaddrinfo(res);
- return ret;
-}
-
-static int run_client(char *dst, char *src)
+static int run_client(void)
{
int i, ret, ret2;
printf("cmatose: starting client\n");
- if (src) {
- ret = get_addr(src, &test.src_in);
+ if (src_addr) {
+ ret = get_addr(src_addr, &test.src_in);
if (ret)
return ret;
}
- ret = get_addr(dst, &test.dst_in);
+ ret = get_addr(dst_addr, &test.dst_in);
if (ret)
return ret;
@@ -570,7 +572,7 @@ static int run_client(char *dst, char *s
printf("cmatose: connecting\n");
for (i = 0; i < connections; i++) {
ret = rdma_resolve_addr(test.nodes[i].cma_id,
- src ? test.src_addr : NULL,
+ src_addr ? test.src_addr : NULL,
test.dst_addr, 2000);
if (ret) {
printf("cmatose: failure getting addr: %d\n", ret);
@@ -597,7 +599,6 @@ static int run_client(char *dst, char *s
}
printf("data transfers complete\n");
-
}
ret = 0;
@@ -611,13 +612,35 @@ out:
int main(int argc, char **argv)
{
- int ret;
+ int op, ret;
- if (argc > 3) {
- printf("usage: %s [server_addr [src_addr]]\n", argv[0]);
- exit(1);
+ while ((op = getopt(argc, argv, "s:b:c:C:S:")) != -1) {
+ switch (op) {
+ case 's':
+ dst_addr = optarg;
+ break;
+ case 'b':
+ src_addr = optarg;
+ break;
+ case 'c':
+ connections = atoi(optarg);
+ break;
+ case 'C':
+ message_count = atoi(optarg);
+ break;
+ case 'S':
+ message_size = atoi(optarg);
+ break;
+ default:
+ printf("usage: %s\n", argv[0]);
+ printf("\t[-s server_address]\n");
+ printf("\t[-b bind_address]\n");
+ printf("\t[-c connections]\n");
+ printf("\t[-C message_count]\n");
+ printf("\t[-S message_size]\n");
+ exit(1);
+ }
}
- is_server = (argc == 1);
test.dst_addr = (struct sockaddr *) &test.dst_in;
test.src_addr = (struct sockaddr *) &test.src_in;
@@ -633,10 +656,10 @@ int main(int argc, char **argv)
if (alloc_nodes())
exit(1);
- if (is_server)
- ret = run_server();
+ if (dst_addr)
+ ret = run_client();
else
- ret = run_client(argv[1], (argc == 3) ? argv[2] : NULL);
+ ret = run_server();
printf("test complete\n");
destroy_nodes();
More information about the general
mailing list