[openib-general] [PATCH] librdmacm: updated librdmacm to work with proposed 2.6.20 kernel CMA
Steve Wise
swise at opengridcomputing.com
Wed Nov 1 14:13:56 PST 2006
Sean,
This patch removes rdma_get/set_option(). Is that what you intended?
On Wed, 2006-10-25 at 13:49 -0700, Sean Hefty wrote:
> Updates the librdmacm to work with ABI version 3, which is the proposed
> kernel changes for inclusion in 2.6.20.
>
> Test programs are also updated.
>
> Signed-off-by: Sean Hefty <sean.hefty at intel.com>
> ---
> Index: include/rdma/rdma_cma_abi.h
> ===================================================================
> --- include/rdma/rdma_cma_abi.h (revision 9192)
> +++ include/rdma/rdma_cma_abi.h (working copy)
> @@ -33,14 +33,15 @@
> #ifndef RDMA_CMA_ABI_H
> #define RDMA_CMA_ABI_H
>
> +#include <infiniband/kern-abi.h>
> #include <infiniband/sa-kern-abi.h>
>
> /*
> * This file must be kept in sync with the kernel's version of rdma_user_cm.h
> */
>
> -#define RDMA_USER_CM_MIN_ABI_VERSION 1
> -#define RDMA_USER_CM_MAX_ABI_VERSION 2
> +#define RDMA_USER_CM_MIN_ABI_VERSION 3
> +#define RDMA_USER_CM_MAX_ABI_VERSION 3
>
> #define RDMA_MAX_PRIVATE_DATA 256
>
> @@ -60,7 +61,7 @@ enum {
> UCMA_CMD_GET_EVENT,
> UCMA_CMD_GET_OPTION,
> UCMA_CMD_SET_OPTION,
> - UCMA_CMD_GET_DST_ATTR,
> + UCMA_CMD_ESTABLISH,
> UCMA_CMD_JOIN_MCAST,
> UCMA_CMD_LEAVE_MCAST
> };
> @@ -71,11 +72,6 @@ struct ucma_abi_cmd_hdr {
> __u16 out;
> };
>
> -struct ucma_abi_create_id_v1 {
> - __u64 uid;
> - __u64 response;
> -};
> -
> struct ucma_abi_create_id {
> __u64 uid;
> __u64 response;
> @@ -133,7 +129,7 @@ struct ucma_abi_query_route_resp {
>
> struct ucma_abi_conn_param {
> __u32 qp_num;
> - __u32 qp_type;
> + __u32 reserved;
> __u8 private_data[RDMA_MAX_PRIVATE_DATA];
> __u8 private_data_len;
> __u8 srq;
> @@ -145,6 +141,15 @@ struct ucma_abi_conn_param {
> __u8 valid;
> };
>
> +struct ucma_abi_ud_param {
> + __u32 qp_num;
> + __u32 qkey;
> + struct ibv_kern_ah_attr ah_attr;
> + __u8 private_data[RDMA_MAX_PRIVATE_DATA];
> + __u8 private_data_len;
> + __u8 reserved[7];
> +};
> +
> struct ucma_abi_connect {
> struct ucma_abi_conn_param conn_param;
> __u32 id;
> @@ -180,25 +185,13 @@ struct ucma_abi_init_qp_attr {
> __u32 qp_state;
> };
>
> -struct ucma_abi_join_mcast {
> - __u32 id;
> - struct sockaddr_in6 addr;
> - __u64 uid;
> -};
> -
> -struct ucma_abi_leave_mcast {
> +struct ucma_abi_establish {
> __u32 id;
> - struct sockaddr_in6 addr;
> -};
> -
> -struct ucma_abi_dst_attr_resp {
> - __u32 remote_qpn;
> - __u32 remote_qkey;
> - struct ibv_kern_ah_attr ah_attr;
> };
>
> -struct ucma_abi_get_dst_attr {
> - __u64 response;
> +struct ucma_abi_join_mcast {
> + __u64 response; /* ucma_abi_create_id_resp */
> + __u64 uid;
> struct sockaddr_in6 addr;
> __u32 id;
> };
> @@ -212,30 +205,10 @@ struct ucma_abi_event_resp {
> __u32 id;
> __u32 event;
> __u32 status;
> - __u8 private_data_len;
> - __u8 reserved[3];
> - __u8 private_data[RDMA_MAX_PRIVATE_DATA];
> -};
> -
> -struct ucma_abi_get_option {
> - __u64 response;
> - __u64 optval;
> - __u32 id;
> - __u32 level;
> - __u32 optname;
> - __u32 optlen;
> -};
> -
> -struct ucma_abi_get_option_resp {
> - __u32 optlen;
> -};
> -
> -struct ucma_abi_set_option {
> - __u64 optval;
> - __u32 id;
> - __u32 level;
> - __u32 optname;
> - __u32 optlen;
> + union {
> + struct ucma_abi_conn_param conn;
> + struct ucma_abi_ud_param ud;
> + } param;
> };
>
> #endif /* RDMA_CMA_ABI_H */
> Index: include/rdma/rdma_cma.h
> ===================================================================
> --- include/rdma/rdma_cma.h (revision 9272)
> +++ include/rdma/rdma_cma.h (working copy)
> @@ -61,11 +61,11 @@ enum rdma_port_space {
> RDMA_PS_UDP = 0x0111,
> };
>
> -/* Protocol levels for get/set options. */
> -enum {
> - RDMA_PROTO_IP = 0,
> - RDMA_PROTO_IB = 1,
> -};
> +/*
> + * Global qkey value for all UD QPs and multicast groups created via the
> + * RDMA CM.
> + */
> +#define RDMA_UD_QKEY 0x01234567
>
> struct ib_addr {
> union ibv_gid sgid;
> @@ -74,8 +74,12 @@ struct ib_addr {
> };
>
> struct rdma_addr {
> - struct sockaddr_in6 src_addr;
> - struct sockaddr_in6 dst_addr;
> + struct sockaddr src_addr;
> + uint8_t src_pad[sizeof(struct sockaddr_storage) -
> + sizeof(struct sockaddr)];
> + struct sockaddr dst_addr;
> + uint8_t dst_pad[sizeof(struct sockaddr_storage) -
> + sizeof(struct sockaddr)];
> union {
> struct ib_addr ibaddr;
> } addr;
> @@ -101,11 +105,25 @@ struct rdma_cm_id {
> uint8_t port_num;
> };
>
> -struct rdma_multicast_data {
> - void *context;
> - struct sockaddr addr;
> - uint8_t pad[sizeof(struct sockaddr_in6) -
> - sizeof(struct sockaddr)];
> +struct rdma_conn_param {
> + const void *private_data;
> + uint8_t private_data_len;
> + uint8_t responder_resources;
> + uint8_t initiator_depth;
> + uint8_t flow_control;
> + uint8_t retry_count; /* ignored when accepting */
> + uint8_t rnr_retry_count;
> + /* Fields below ignored if a QP is created on the rdma_cm_id. */
> + uint8_t srq;
> + uint32_t qp_num;
> +};
> +
> +struct rdma_ud_param {
> + const void *private_data;
> + uint8_t private_data_len;
> + struct ibv_ah_attr ah_attr;
> + uint32_t qp_num;
> + uint32_t qkey;
> };
>
> struct rdma_cm_event {
> @@ -113,8 +131,10 @@ struct rdma_cm_event {
> struct rdma_cm_id *listen_id;
> enum rdma_cm_event_type event;
> int status;
> - void *private_data;
> - uint8_t private_data_len;
> + union {
> + struct rdma_conn_param conn;
> + struct rdma_ud_param ud;
> + } param;
> };
>
> /**
> @@ -206,20 +226,6 @@ int rdma_create_qp(struct rdma_cm_id *id
> */
> void rdma_destroy_qp(struct rdma_cm_id *id);
>
> -struct rdma_conn_param {
> - const void *private_data;
> - uint8_t private_data_len;
> - uint8_t responder_resources;
> - uint8_t initiator_depth;
> - uint8_t flow_control;
> - uint8_t retry_count; /* ignored when accepting */
> - uint8_t rnr_retry_count;
> - /* Fields below ignored if a QP is created on the rdma_cm_id. */
> - uint8_t srq;
> - uint32_t qp_num;
> - enum ibv_qp_type qp_type;
> -};
> -
> /**
> * rdma_connect - Initiate an active connection request.
> *
> @@ -251,6 +257,16 @@ int rdma_reject(struct rdma_cm_id *id, c
> uint8_t private_data_len);
>
> /**
> + * rdma_establish - Forces a connection state to established.
> + * @id: Connection identifier to transition to established.
> + *
> + * This routine should be invoked by users who receive messages on a
> + * QP before being notified that the connection has been established by the
> + * RDMA CM.
> + */
> +int rdma_establish(struct rdma_cm_id *id);
> +
> +/**
> * rdma_disconnect - This function disconnects the associated QP and
> * transitions it into the error state.
> */
> @@ -298,40 +314,17 @@ int rdma_get_cm_event(struct rdma_event_
> */
> int rdma_ack_cm_event(struct rdma_cm_event *event);
>
> -/**
> - * rdma_get_option - Retrieve options for an rdma_cm_id.
> - * @id: Communication identifier to retrieve option for.
> - * @level: Protocol level of the option to retrieve.
> - * @optname: Name of the option to retrieve.
> - * @optval: Buffer to receive the returned options.
> - * @optlen: On input, the size of the %optval buffer. On output, the
> - * size of the returned data.
> - */
> -int rdma_get_option(struct rdma_cm_id *id, int level, int optname,
> - void *optval, size_t *optlen);
> -
> -/**
> - * rdma_set_option - Set options for an rdma_cm_id.
> - * @id: Communication identifier to set option for.
> - * @level: Protocol level of the option to set.
> - * @optname: Name of the option to set.
> - * @optval: Reference to the option data.
> - * @optlen: The size of the %optval buffer.
> - */
> -int rdma_set_option(struct rdma_cm_id *id, int level, int optname,
> - void *optval, size_t optlen);
> -
> static inline uint16_t rdma_get_src_port(struct rdma_cm_id *id)
> {
> - return id->route.addr.src_addr.sin6_family == PF_INET6 ?
> - id->route.addr.src_addr.sin6_port :
> + return id->route.addr.src_addr.sa_family == PF_INET6 ?
> + ((struct sockaddr_in6 *) &id->route.addr.src_addr)->sin6_port :
> ((struct sockaddr_in *) &id->route.addr.src_addr)->sin_port;
> }
>
> static inline uint16_t rdma_get_dst_port(struct rdma_cm_id *id)
> {
> - return id->route.addr.dst_addr.sin6_family == PF_INET6 ?
> - id->route.addr.dst_addr.sin6_port :
> + return id->route.addr.dst_addr.sa_family == PF_INET6 ?
> + ((struct sockaddr_in6 *) &id->route.addr.dst_addr)->sin6_port :
> ((struct sockaddr_in *) &id->route.addr.dst_addr)->sin_port;
> }
>
> Index: src/cma.c
> ===================================================================
> --- src/cma.c (revision 9696)
> +++ src/cma.c (working copy)
> @@ -1,5 +1,5 @@
> /*
> - * Copyright (c) 2005 Intel Corporation. All rights reserved.
> + * Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
> *
> * This software is available to you under a choice of one of two
> * licenses. You may choose to be licensed under the terms of the GNU
> @@ -54,7 +54,6 @@
> #include <infiniband/marshall.h>
> #include <rdma/rdma_cma.h>
> #include <rdma/rdma_cma_abi.h>
> -#include <rdma/rdma_cma_ib.h>
>
> #define PFX "librdmacm: "
>
> @@ -116,6 +115,28 @@ struct cma_id_private {
> pthread_cond_t cond;
> pthread_mutex_t mut;
> uint32_t handle;
> + struct cma_multicast *mc_list;
> +};
> +
> +struct cma_multicast {
> + struct cma_multicast *next;
> + struct cma_id_private *id_priv;
> + void *context;
> + int events_completed;
> + pthread_cond_t cond;
> + uint32_t handle;
> + union ibv_gid mgid;
> + uint16_t mlid;
> + struct sockaddr addr;
> + uint8_t pad[sizeof(struct sockaddr_in6) -
> + sizeof(struct sockaddr)];
> +};
> +
> +struct cma_event {
> + struct rdma_cm_event event;
> + uint8_t private_data[RDMA_MAX_PRIVATE_DATA];
> + struct cma_id_private *id_priv;
> + struct cma_multicast *mc;
> };
>
> static struct cma_device *cma_dev_array;
> @@ -335,41 +356,6 @@ err: ucma_free_id(id_priv);
> return NULL;
> }
>
> -static int ucma_create_id_v1(struct rdma_event_channel *channel,
> - struct rdma_cm_id **id, void *context,
> - enum rdma_port_space ps)
> -{
> - struct ucma_abi_create_id_resp *resp;
> - struct ucma_abi_create_id_v1 *cmd;
> - struct cma_id_private *id_priv;
> - void *msg;
> - int ret, size;
> -
> - if (ps != RDMA_PS_TCP) {
> - fprintf(stderr, "librdmacm: Kernel ABI does not support "
> - "requested port space.\n");
> - return -EPROTONOSUPPORT;
> - }
> -
> - id_priv = ucma_alloc_id(channel, context, ps);
> - if (!id_priv)
> - return -ENOMEM;
> -
> - CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_CREATE_ID, size);
> - cmd->uid = (uintptr_t) id_priv;
> -
> - ret = write(channel->fd, msg, size);
> - if (ret != size)
> - goto err;
> -
> - id_priv->handle = resp->id;
> - *id = &id_priv->id;
> - return 0;
> -
> -err: ucma_free_id(id_priv);
> - return ret;
> -}
> -
> int rdma_create_id(struct rdma_event_channel *channel,
> struct rdma_cm_id **id, void *context,
> enum rdma_port_space ps)
> @@ -384,9 +370,6 @@ int rdma_create_id(struct rdma_event_cha
> if (ret)
> return ret;
>
> - if (abi_ver == 1)
> - return ucma_create_id_v1(channel, id, context, ps);
> -
> id_priv = ucma_alloc_id(channel, context, ps);
> if (!id_priv)
> return -ENOMEM;
> @@ -492,9 +475,9 @@ static int ucma_query_route(struct rdma_
> sizeof id->route.addr.addr.ibaddr.dgid);
> id->route.addr.addr.ibaddr.pkey = resp->ib_route[0].pkey;
> memcpy(&id->route.addr.src_addr, &resp->src_addr,
> - sizeof id->route.addr.src_addr);
> + sizeof resp->src_addr);
> memcpy(&id->route.addr.dst_addr, &resp->dst_addr,
> - sizeof id->route.addr.dst_addr);
> + sizeof resp->dst_addr);
>
> if (!id_priv->cma_dev && resp->node_guid) {
> ret = ucma_get_device(id_priv, resp->node_guid);
> @@ -696,7 +679,7 @@ static int ucma_init_ib_qp(struct cma_id
>
> qp_attr.port_num = id_priv->id.port_num;
> qp_attr.qp_state = IBV_QPS_INIT;
> - qp_attr.qp_access_flags = IBV_ACCESS_LOCAL_WRITE;
> + qp_attr.qp_access_flags = 0;
> return ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_ACCESS_FLAGS |
> IBV_QP_PKEY_INDEX | IBV_QP_PORT);
> }
> @@ -767,11 +750,9 @@ void rdma_destroy_qp(struct rdma_cm_id *
>
> static void ucma_copy_conn_param_to_kern(struct ucma_abi_conn_param *dst,
> struct rdma_conn_param *src,
> - uint32_t qp_num,
> - enum ibv_qp_type qp_type, uint8_t srq)
> + uint32_t qp_num, uint8_t srq)
> {
> dst->qp_num = qp_num;
> - dst->qp_type = qp_type;
> dst->srq = srq;
> dst->responder_resources = src->responder_resources;
> dst->initiator_depth = src->initiator_depth;
> @@ -799,12 +780,11 @@ int rdma_connect(struct rdma_cm_id *id,
> cmd->id = id_priv->handle;
> if (id->qp)
> ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
> - id->qp->qp_num, id->qp->qp_type,
> + id->qp->qp_num,
> (id->qp->srq != NULL));
> else
> ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
> conn_param->qp_num,
> - conn_param->qp_type,
> conn_param->srq);
>
> ret = write(id->channel->fd, msg, size);
> @@ -852,12 +832,11 @@ int rdma_accept(struct rdma_cm_id *id, s
> cmd->uid = (uintptr_t) id_priv;
> if (id->qp)
> ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
> - id->qp->qp_num, id->qp->qp_type,
> + id->qp->qp_num,
> (id->qp->srq != NULL));
> else
> ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
> conn_param->qp_num,
> - conn_param->qp_type,
> conn_param->srq);
>
> ret = write(id->channel->fd, msg, size);
> @@ -894,6 +873,24 @@ int rdma_reject(struct rdma_cm_id *id, c
> return 0;
> }
>
> +int rdma_establish(struct rdma_cm_id *id)
> +{
> + struct ucma_abi_establish *cmd;
> + struct cma_id_private *id_priv;
> + void *msg;
> + int ret, size;
> +
> + CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_ESTABLISH, size);
> +
> + id_priv = container_of(id, struct cma_id_private, id);
> + cmd->id = id_priv->handle;
> + ret = write(id->channel->fd, msg, size);
> + if (ret != size)
> + return (ret > 0) ? -ENODATA : ret;
> +
> + return 0;
> +}
> +
> int rdma_disconnect(struct rdma_cm_id *id)
> {
> struct ucma_abi_disconnect *cmd;
> @@ -929,74 +926,102 @@ int rdma_join_multicast(struct rdma_cm_i
> void *context)
> {
> struct ucma_abi_join_mcast *cmd;
> + struct ucma_abi_create_id_resp *resp;
> struct cma_id_private *id_priv;
> + struct cma_multicast *mc, **pos;
> void *msg;
> int ret, size, addrlen;
>
> + id_priv = container_of(id, struct cma_id_private, id);
> addrlen = ucma_addrlen(addr);
> if (!addrlen)
> return -EINVAL;
>
> - CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_JOIN_MCAST, size);
> - id_priv = container_of(id, struct cma_id_private, id);
> + mc = malloc(sizeof *mc);
> + if (!mc)
> + return -ENOMEM;
> +
> + memset(mc, 0, sizeof *mc);
> + mc->context = context;
> + mc->id_priv = id_priv;
> + memcpy(&mc->addr, addr, addrlen);
> + if (pthread_cond_init(&id_priv->cond, NULL)) {
> + ret = -1;
> + goto err1;
> + }
> +
> + pthread_mutex_lock(&id_priv->mut);
> + mc->next = id_priv->mc_list;
> + id_priv->mc_list = mc;
> + pthread_mutex_unlock(&id_priv->mut);
> +
> + CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_JOIN_MCAST, size);
> cmd->id = id_priv->handle;
> memcpy(&cmd->addr, addr, addrlen);
> - cmd->uid = (uintptr_t) context;
> + cmd->uid = (uintptr_t) mc;
>
> ret = write(id->channel->fd, msg, size);
> - if (ret != size)
> - return (ret > 0) ? -ENODATA : ret;
> + if (ret != size) {
> + ret = (ret > 0) ? -ENODATA : ret;
> + goto err2;
> + }
>
> + mc->handle = resp->id;
> return 0;
> +err2:
> + pthread_mutex_lock(&id_priv->mut);
> + for (pos = &id_priv->mc_list; *pos != mc; pos = &(*pos)->next)
> + ;
> + *pos = mc->next;
> + pthread_mutex_unlock(&id_priv->mut);
> +err1:
> + free(mc);
> + return ret;
> }
>
> int rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
> {
> - struct ucma_abi_leave_mcast *cmd;
> + struct ucma_abi_destroy_id *cmd;
> + struct ucma_abi_destroy_id_resp *resp;
> struct cma_id_private *id_priv;
> + struct cma_multicast *mc, **pos;
> void *msg;
> int ret, size, addrlen;
> - struct ibv_ah_attr ah_attr;
> - uint32_t qp_info;
>
> addrlen = ucma_addrlen(addr);
> if (!addrlen)
> return -EINVAL;
>
> - CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_LEAVE_MCAST, size);
> id_priv = container_of(id, struct cma_id_private, id);
> - cmd->id = id_priv->handle;
> - memcpy(&cmd->addr, addr, addrlen);
> + pthread_mutex_lock(&id_priv->mut);
> + for (pos = &id_priv->mc_list; *pos; pos = &(*pos)->next)
> + if (!memcmp(&(*pos)->addr, addr, addrlen))
> + break;
>
> - if (id->qp) {
> - ret = rdma_get_dst_attr(id, addr, &ah_attr, &qp_info, &qp_info);
> - if (ret)
> - goto out;
> + mc = *pos;
> + if (*pos)
> + *pos = mc->next;
> + pthread_mutex_unlock(&id_priv->mut);
> + if (!mc)
> + return -EADDRNOTAVAIL;
>
> - ret = ibv_detach_mcast(id->qp, &ah_attr.grh.dgid, ah_attr.dlid);
> - if (ret)
> - goto out;
> - }
> + if (id->qp)
> + ibv_detach_mcast(id->qp, &mc->mgid, mc->mlid);
>
> + CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_LEAVE_MCAST, size);
> + cmd->id = mc->handle;
> +
> ret = write(id->channel->fd, msg, size);
> if (ret != size)
> ret = (ret > 0) ? -ENODATA : ret;
> -out:
> - return ret;
> -}
>
> -static void ucma_copy_event_from_kern(struct rdma_cm_event *dst,
> - struct ucma_abi_event_resp *src)
> -{
> - dst->event = src->event;
> - dst->status = src->status;
> - dst->private_data_len = src->private_data_len;
> - if (src->private_data_len) {
> - dst->private_data = dst + 1;
> - memcpy(dst->private_data, src->private_data,
> - src->private_data_len);
> - } else
> - dst->private_data = NULL;
> + pthread_mutex_lock(&id_priv->mut);
> + while (mc->events_completed < resp->events_reported)
> + pthread_cond_wait(&mc->cond, &id_priv->mut);
> + pthread_mutex_unlock(&id_priv->mut);
> +
> + free(mc);
> + return ret;
> }
>
> static void ucma_complete_event(struct cma_id_private *id_priv)
> @@ -1007,38 +1032,49 @@ static void ucma_complete_event(struct c
> pthread_mutex_unlock(&id_priv->mut);
> }
>
> +static void ucma_complete_mc_event(struct cma_multicast *mc)
> +{
> + pthread_mutex_lock(&mc->id_priv->mut);
> + mc->events_completed++;
> + pthread_cond_signal(&mc->cond);
> + mc->id_priv->events_completed++;
> + pthread_cond_signal(&mc->id_priv->cond);
> + pthread_mutex_unlock(&mc->id_priv->mut);
> +}
> +
> int rdma_ack_cm_event(struct rdma_cm_event *event)
> {
> - struct rdma_cm_id *id;
> + struct cma_event *evt;
>
> if (!event)
> return -EINVAL;
>
> - id = (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) ?
> - event->listen_id : event->id;
> + evt = container_of(event, struct cma_event, event);
>
> - ucma_complete_event(container_of(id, struct cma_id_private, id));
> - free(event);
> + if (evt->mc)
> + ucma_complete_mc_event(evt->mc);
> + else
> + ucma_complete_event(evt->id_priv);
> + free(evt);
> return 0;
> }
>
> -static int ucma_process_conn_req(struct rdma_cm_event *event,
> +static int ucma_process_conn_req(struct cma_event *evt,
> uint32_t handle)
> {
> - struct cma_id_private *listen_id_priv, *id_priv;
> + struct cma_id_private *id_priv;
> int ret;
>
> - listen_id_priv = container_of(event->id, struct cma_id_private, id);
> - id_priv = ucma_alloc_id(event->id->channel, event->id->context,
> - event->id->ps);
> + id_priv = ucma_alloc_id(evt->id_priv->id.channel,
> + evt->id_priv->id.context, evt->id_priv->id.ps);
> if (!id_priv) {
> - ucma_destroy_kern_id(event->id->channel->fd, handle);
> + ucma_destroy_kern_id(evt->id_priv->id.channel->fd, handle);
> ret = -ENOMEM;
> goto err;
> }
>
> - event->listen_id = event->id;
> - event->id = &id_priv->id;
> + evt->event.listen_id = &evt->id_priv->id;
> + evt->event.id = &id_priv->id;
> id_priv->handle = handle;
>
> ret = ucma_query_route(&id_priv->id);
> @@ -1049,7 +1085,7 @@ static int ucma_process_conn_req(struct
>
> return 0;
> err:
> - ucma_complete_event(listen_id_priv);
> + ucma_complete_event(evt->id_priv);
> return ret;
> }
>
> @@ -1093,34 +1129,54 @@ static int ucma_process_establish(struct
> return ret;
> }
>
> -static void ucma_process_mcast(struct rdma_cm_id *id, struct rdma_cm_event *evt)
> +static int ucma_process_join(struct cma_event *evt)
> {
> - struct ucma_abi_join_mcast kmc_data;
> - struct rdma_multicast_data *mc_data;
> - struct ibv_ah_attr ah_attr;
> - uint32_t qp_info;
> -
> - kmc_data = *(struct ucma_abi_join_mcast *) evt->private_data;
> -
> - mc_data = evt->private_data;
> - mc_data->context = (void *) (uintptr_t) kmc_data.uid;
> - memcpy(&mc_data->addr, &kmc_data.addr,
> - ucma_addrlen((struct sockaddr *) &kmc_data.addr));
> -
> - if (evt->status || !id->qp)
> - return;
> -
> - evt->status = rdma_get_dst_attr(id, &mc_data->addr, &ah_attr,
> - &qp_info, &qp_info);
> - if (evt->status)
> - goto err;
> + evt->mc->mgid = evt->event.param.ud.ah_attr.grh.dgid;
> + evt->mc->mlid = evt->event.param.ud.ah_attr.dlid;
>
> - evt->status = ibv_attach_mcast(id->qp, &ah_attr.grh.dgid, ah_attr.dlid);
> - if (evt->status)
> - goto err;
> - return;
> -err:
> - evt->event = RDMA_CM_EVENT_MULTICAST_ERROR;
> + if (evt->id_priv->id.qp)
> + return ibv_attach_mcast(evt->id_priv->id.qp,
> + &evt->mc->mgid, evt->mc->mlid);
> + else
> + return 0;
> +}
> +
> +static void ucma_copy_conn_event(struct cma_event *event,
> + struct ucma_abi_conn_param *src)
> +{
> + struct rdma_conn_param *dst = &event->event.param.conn;
> +
> + dst->private_data_len = src->private_data_len;
> + if (src->private_data_len) {
> + dst->private_data = &event->private_data;
> + memcpy(&event->private_data, src->private_data,
> + src->private_data_len);
> + }
> +
> + dst->responder_resources = src->responder_resources;
> + dst->initiator_depth = src->initiator_depth;
> + dst->flow_control = src->flow_control;
> + dst->retry_count = src->retry_count;
> + dst->rnr_retry_count = src->rnr_retry_count;
> + dst->srq = src->srq;
> + dst->qp_num = src->qp_num;
> +}
> +
> +static void ucma_copy_ud_event(struct cma_event *event,
> + struct ucma_abi_ud_param *src)
> +{
> + struct rdma_ud_param *dst = &event->event.param.ud;
> +
> + dst->private_data_len = src->private_data_len;
> + if (src->private_data_len) {
> + dst->private_data = &event->private_data;
> + memcpy(&event->private_data, src->private_data,
> + src->private_data_len);
> + }
> +
> + ibv_copy_ah_attr_from_kern(&dst->ah_attr, &src->ah_attr);
> + dst->qp_num = src->qp_num;
> + dst->qkey = src->qkey;
> }
>
> int rdma_get_cm_event(struct rdma_event_channel *channel,
> @@ -1128,8 +1184,7 @@ int rdma_get_cm_event(struct rdma_event_
> {
> struct ucma_abi_event_resp *resp;
> struct ucma_abi_get_event *cmd;
> - struct cma_id_private *id_priv;
> - struct rdma_cm_event *evt;
> + struct cma_event *evt;
> void *msg;
> int ret, size;
>
> @@ -1140,155 +1195,119 @@ int rdma_get_cm_event(struct rdma_event_
> if (!event)
> return -EINVAL;
>
> - evt = malloc(sizeof *evt + RDMA_MAX_PRIVATE_DATA);
> + evt = malloc(sizeof *evt);
> if (!evt)
> return -ENOMEM;
>
> retry:
> + memset(evt, 0, sizeof *evt);
> CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_GET_EVENT, size);
> ret = write(channel->fd, msg, size);
> if (ret != size) {
> free(evt);
> return (ret > 0) ? -ENODATA : ret;
> }
> -
> - id_priv = (void *) (uintptr_t) resp->uid;
> - evt->id = &id_priv->id;
> - ucma_copy_event_from_kern(evt, resp);
>
> - switch (evt->event) {
> + evt->event.event = resp->event;
> + switch (resp->event) {
> case RDMA_CM_EVENT_ADDR_RESOLVED:
> - evt->status = ucma_query_route(&id_priv->id);
> - if (evt->status)
> - evt->event = RDMA_CM_EVENT_ADDR_ERROR;
> + evt->id_priv = (void *) (uintptr_t) resp->uid;
> + evt->event.id = &evt->id_priv->id;
> + evt->event.status = ucma_query_route(&evt->id_priv->id);
> + if (evt->event.status)
> + evt->event.event = RDMA_CM_EVENT_ADDR_ERROR;
> break;
> case RDMA_CM_EVENT_ROUTE_RESOLVED:
> - evt->status = ucma_query_route(&id_priv->id);
> - if (evt->status)
> - evt->event = RDMA_CM_EVENT_ROUTE_ERROR;
> + evt->id_priv = (void *) (uintptr_t) resp->uid;
> + evt->event.id = &evt->id_priv->id;
> + evt->event.status = ucma_query_route(&evt->id_priv->id);
> + if (evt->event.status)
> + evt->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
> break;
> case RDMA_CM_EVENT_CONNECT_REQUEST:
> + evt->id_priv = (void *) (uintptr_t) resp->uid;
> + if (evt->id_priv->id.ps == RDMA_PS_TCP)
> + ucma_copy_conn_event(evt, &resp->param.conn);
> + else
> + ucma_copy_ud_event(evt, &resp->param.ud);
> +
> ret = ucma_process_conn_req(evt, resp->id);
> if (ret)
> goto retry;
> break;
> case RDMA_CM_EVENT_CONNECT_RESPONSE:
> - evt->status = ucma_process_conn_resp(id_priv);
> - if (!evt->status)
> - evt->event = RDMA_CM_EVENT_ESTABLISHED;
> + evt->id_priv = (void *) (uintptr_t) resp->uid;
> + evt->event.id = &evt->id_priv->id;
> + ucma_copy_conn_event(evt, &resp->param.conn);
> + evt->event.status = ucma_process_conn_resp(evt->id_priv);
> + if (!evt->event.status)
> + evt->event.event = RDMA_CM_EVENT_ESTABLISHED;
> else {
> - evt->event = RDMA_CM_EVENT_CONNECT_ERROR;
> - id_priv->connect_error = 1;
> + evt->event.event = RDMA_CM_EVENT_CONNECT_ERROR;
> + evt->id_priv->connect_error = 1;
> }
> break;
> case RDMA_CM_EVENT_ESTABLISHED:
> - if (id_priv->id.ps == RDMA_PS_UDP)
> + evt->id_priv = (void *) (uintptr_t) resp->uid;
> + evt->event.id = &evt->id_priv->id;
> + if (evt->id_priv->id.ps == RDMA_PS_UDP) {
> + ucma_copy_ud_event(evt, &resp->param.ud);
> break;
> + }
>
> - evt->status = ucma_process_establish(&id_priv->id);
> - if (evt->status) {
> - evt->event = RDMA_CM_EVENT_CONNECT_ERROR;
> - id_priv->connect_error = 1;
> + ucma_copy_conn_event(evt, &resp->param.conn);
> + evt->event.status = ucma_process_establish(&evt->id_priv->id);
> + if (evt->event.status) {
> + evt->event.event = RDMA_CM_EVENT_CONNECT_ERROR;
> + evt->id_priv->connect_error = 1;
> }
> break;
> case RDMA_CM_EVENT_REJECTED:
> - if (id_priv->connect_error) {
> - ucma_complete_event(id_priv);
> + evt->id_priv = (void *) (uintptr_t) resp->uid;
> + if (evt->id_priv->connect_error) {
> + ucma_complete_event(evt->id_priv);
> goto retry;
> }
> - ucma_modify_qp_err(evt->id);
> + evt->event.id = &evt->id_priv->id;
> + ucma_copy_conn_event(evt, &resp->param.conn);
> + ucma_modify_qp_err(evt->event.id);
> break;
> case RDMA_CM_EVENT_DISCONNECTED:
> - if (id_priv->connect_error) {
> - ucma_complete_event(id_priv);
> + evt->id_priv = (void *) (uintptr_t) resp->uid;
> + if (evt->id_priv->connect_error) {
> + ucma_complete_event(evt->id_priv);
> goto retry;
> }
> + evt->event.id = &evt->id_priv->id;
> + ucma_copy_conn_event(evt, &resp->param.conn);
> break;
> case RDMA_CM_EVENT_MULTICAST_JOIN:
> + evt->mc = (void *) (uintptr_t) resp->uid;
> + evt->id_priv = evt->mc->id_priv;
> + evt->event.id = &evt->id_priv->id;
> + ucma_copy_ud_event(evt, &resp->param.ud);
> + evt->event.param.ud.private_data = evt->mc->context;
> + evt->event.status = ucma_process_join(evt);
> + if (evt->event.status)
> + evt->event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
> + break;
> case RDMA_CM_EVENT_MULTICAST_ERROR:
> - ucma_process_mcast(&id_priv->id, evt);
> + evt->mc = (void *) (uintptr_t) resp->uid;
> + evt->id_priv = evt->mc->id_priv;
> + evt->event.id = &evt->id_priv->id;
> + evt->event.status = resp->status;
> + evt->event.param.ud.private_data = evt->mc->context;
> break;
> default:
> + evt->id_priv = (void *) (uintptr_t) resp->uid;
> + evt->event.id = &evt->id_priv->id;
> + if (evt->id_priv->id.ps == RDMA_PS_TCP)
> + ucma_copy_conn_event(evt, &resp->param.conn);
> + else
> + ucma_copy_ud_event(evt, &resp->param.ud);
> break;
> }
>
> - *event = evt;
> - return 0;
> -}
> -
> -int rdma_get_option(struct rdma_cm_id *id, int level, int optname,
> - void *optval, size_t *optlen)
> -{
> - struct ucma_abi_get_option_resp *resp;
> - struct ucma_abi_get_option *cmd;
> - struct cma_id_private *id_priv;
> - void *msg;
> - int ret, size;
> -
> - CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_GET_OPTION, size);
> - id_priv = container_of(id, struct cma_id_private, id);
> - cmd->id = id_priv->handle;
> - cmd->optval = (uintptr_t) optval;
> - cmd->level = level;
> - cmd->optname = optname;
> - cmd->optlen = *optlen;
> -
> - ret = write(id->channel->fd, msg, size);
> - if (ret != size)
> - return (ret > 0) ? -ENODATA : ret;
> -
> - *optlen = resp->optlen;
> - return 0;
> -}
> -
> -int rdma_set_option(struct rdma_cm_id *id, int level, int optname,
> - void *optval, size_t optlen)
> -{
> - struct ucma_abi_set_option *cmd;
> - struct cma_id_private *id_priv;
> - void *msg;
> - int ret, size;
> -
> - CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_SET_OPTION, size);
> - id_priv = container_of(id, struct cma_id_private, id);
> - cmd->id = id_priv->handle;
> - cmd->optval = (uintptr_t) optval;
> - cmd->level = level;
> - cmd->optname = optname;
> - cmd->optlen = optlen;
> -
> - ret = write(id->channel->fd, msg, size);
> - if (ret != size)
> - return (ret > 0) ? -ENODATA : ret;
> -
> - return 0;
> -}
> -
> -int rdma_get_dst_attr(struct rdma_cm_id *id, struct sockaddr *addr,
> - struct ibv_ah_attr *ah_attr, uint32_t *remote_qpn,
> - uint32_t *remote_qkey)
> -{
> - struct ucma_abi_dst_attr_resp *resp;
> - struct ucma_abi_get_dst_attr *cmd;
> - struct cma_id_private *id_priv;
> - void *msg;
> - int ret, size, addrlen;
> -
> - addrlen = ucma_addrlen(addr);
> - if (!addrlen)
> - return -EINVAL;
> -
> - CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_GET_DST_ATTR, size);
> - id_priv = container_of(id, struct cma_id_private, id);
> - cmd->id = id_priv->handle;
> - memcpy(&cmd->addr, addr, addrlen);
> -
> - ret = write(id->channel->fd, msg, size);
> - if (ret != size)
> - return (ret > 0) ? -ENODATA : ret;
> -
> - ibv_copy_ah_attr_from_kern(ah_attr, &resp->ah_attr);
> - *remote_qpn = resp->remote_qpn;
> - *remote_qkey = resp->remote_qkey;
> + *event = &evt->event;
> return 0;
> }
> Index: Makefile.am
> ===================================================================
> --- Makefile.am (revision 9192)
> +++ Makefile.am (working copy)
> @@ -31,12 +31,10 @@ examples_mckey_LDADD = $(top_builddir)/s
> librdmacmincludedir = $(includedir)/rdma
>
> librdmacminclude_HEADERS = include/rdma/rdma_cma_abi.h \
> - include/rdma/rdma_cma.h \
> - include/rdma/rdma_cma_ib.h
> + include/rdma/rdma_cma.h
>
> EXTRA_DIST = include/rdma/rdma_cma_abi.h \
> include/rdma/rdma_cma.h \
> - include/rdma/rdma_cma_ib.h \
> src/librdmacm.map \
> librdmacm.spec.in
>
> Index: examples/mckey.c
> ===================================================================
> --- examples/mckey.c (revision 9208)
> +++ examples/mckey.c (working copy)
> @@ -1,5 +1,5 @@
> /*
> - * Copyright (c) 2005 Intel Corporation. All rights reserved.
> + * Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
> *
> * This software is available to you under a choice of one of two
> * licenses. You may choose to be licensed under the terms of the GNU
> @@ -42,9 +42,9 @@
> #include <netdb.h>
> #include <byteswap.h>
> #include <unistd.h>
> +#include <getopt.h>
>
> #include <rdma/rdma_cma.h>
> -#include <rdma/rdma_cma_ib.h>
>
> struct cmatest_node {
> int id;
> @@ -76,6 +76,8 @@ static int connections = 1;
> static int message_size = 100;
> static int message_count = 10;
> static int is_sender;
> +static char *dst_addr;
> +static char *src_addr;
>
> static int create_message(struct cmatest_node *node)
> {
> @@ -239,19 +241,12 @@ err:
> return ret;
> }
>
> -static int join_handler(struct cmatest_node *node)
> +static int join_handler(struct cmatest_node *node,
> + struct rdma_ud_param *param)
> {
> - struct ibv_ah_attr ah_attr;
> - int ret;
> -
> - ret = rdma_get_dst_attr(node->cma_id, test.dst_addr, &ah_attr,
> - &node->remote_qpn, &node->remote_qkey);
> - if (ret) {
> - printf("mckey: failure getting destination attributes\n");
> - goto err;
> - }
> -
> - node->ah = ibv_create_ah(node->pd, &ah_attr);
> + node->remote_qpn = param->qp_num;
> + node->remote_qkey = param->qkey;
> + node->ah = ibv_create_ah(node->pd, ¶m->ah_attr);
> if (!node->ah) {
> printf("mckey: failure creating address handle\n");
> goto err;
> @@ -262,7 +257,7 @@ static int join_handler(struct cmatest_n
> return 0;
> err:
> connect_error();
> - return ret;
> + return -1;
> }
>
> static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
> @@ -274,7 +269,7 @@ static int cma_handler(struct rdma_cm_id
> ret = addr_handler(cma_id->context);
> break;
> case RDMA_CM_EVENT_MULTICAST_JOIN:
> - ret = join_handler(cma_id->context);
> + ret = join_handler(cma_id->context, &event->param.ud);
> break;
> case RDMA_CM_EVENT_ADDR_ERROR:
> case RDMA_CM_EVENT_ROUTE_ERROR:
> @@ -411,18 +406,21 @@ out:
> return ret;
> }
>
> -static int run(char *dst, char *src)
> +static int run(void)
> {
> int i, ret;
>
> - printf("mckey: starting client\n");
> - if (src) {
> - ret = get_addr(src, &test.src_in);
> + if (is_sender)
> + printf("mckey: starting client\n");
> + else
> + printf("mckey: starting server\n");
> + if (src_addr) {
> + ret = get_addr(src_addr, &test.src_in);
> if (ret)
> return ret;
> }
>
> - ret = get_addr(dst, &test.dst_in);
> + ret = get_addr(dst_addr, &test.dst_in);
> if (ret)
> return ret;
>
> @@ -431,7 +429,7 @@ static int run(char *dst, char *src)
> printf("mckey: joining\n");
> for (i = 0; i < connections; i++) {
> ret = rdma_resolve_addr(test.nodes[i].cma_id,
> - src ? test.src_addr : NULL,
> + src_addr ? test.src_addr : NULL,
> test.dst_addr, 2000);
> if (ret) {
> printf("mckey: failure getting addr: %d\n", ret);
> @@ -472,14 +470,39 @@ out:
>
> int main(int argc, char **argv)
> {
> - int ret;
> + int op, ret;
>
> - if (argc < 3 || argc > 4) {
> - printf("usage: %s {s[end] | r[ecv]} mcast_addr [bind_addr]]\n",
> - argv[0]);
> - exit(1);
> + while ((op = getopt(argc, argv, "m:sb:c:C:S:")) != -1) {
> + switch (op) {
> + case 'm':
> + dst_addr = optarg;
> + break;
> + case 's':
> + is_sender = 1;
> + break;
> + case 'b':
> + src_addr = optarg;
> + break;
> + case 'c':
> + connections = atoi(optarg);
> + break;
> + case 'C':
> + message_count = atoi(optarg);
> + break;
> + case 'S':
> + message_size = atoi(optarg);
> + break;
> + default:
> + printf("usage: %s\n", argv[0]);
> + printf("\t-m multicast_address\n");
> + printf("\t[-s(ender)]\n");
> + printf("\t[-b bind_address]\n");
> + printf("\t[-c connections]\n");
> + printf("\t[-C message_count]\n");
> + printf("\t[-S message_size]\n");
> + exit(1);
> + }
> }
> - is_sender = (argv[1][0] == 's');
>
> test.dst_addr = (struct sockaddr *) &test.dst_in;
> test.src_addr = (struct sockaddr *) &test.src_in;
> @@ -494,7 +517,7 @@ int main(int argc, char **argv)
> if (alloc_nodes())
> exit(1);
>
> - ret = run(argv[2], (argc == 4) ? argv[3] : NULL);
> + ret = run();
>
> printf("test complete\n");
> destroy_nodes();
> Index: examples/udaddy.c
> ===================================================================
> --- examples/udaddy.c (revision 9208)
> +++ examples/udaddy.c (working copy)
> @@ -1,5 +1,5 @@
> /*
> - * Copyright (c) 2005 Intel Corporation. All rights reserved.
> + * Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
> *
> * This software is available to you under a choice of one of two
> * licenses. You may choose to be licensed under the terms of the GNU
> @@ -41,15 +41,9 @@
> #include <sys/socket.h>
> #include <netdb.h>
> #include <byteswap.h>
> +#include <getopt.h>
>
> #include <rdma/rdma_cma.h>
> -#include <rdma/rdma_cma_ib.h>
> -
> -/*
> - * To execute:
> - * Server: udaddy
> - * Client: udaddy [server_addr [src_addr]]
> - */
>
> struct cmatest_node {
> int id;
> @@ -80,7 +74,8 @@ static struct cmatest test;
> static int connections = 1;
> static int message_size = 100;
> static int message_count = 10;
> -static int is_server;
> +static char *dst_addr;
> +static char *src_addr;
>
> static int create_message(struct cmatest_node *node)
> {
> @@ -246,7 +241,6 @@ static int route_handler(struct cmatest_
>
> memset(&conn_param, 0, sizeof conn_param);
> conn_param.qp_num = node->cma_id->qp->qp_num;
> - conn_param.qp_type = node->cma_id->qp->qp_type;
> conn_param.retry_count = 5;
> ret = rdma_connect(node->cma_id, &conn_param);
> if (ret) {
> @@ -284,7 +278,6 @@ static int connect_handler(struct rdma_c
>
> memset(&conn_param, 0, sizeof conn_param);
> conn_param.qp_num = node->cma_id->qp->qp_num;
> - conn_param.qp_type = node->cma_id->qp->qp_type;
> ret = rdma_accept(node->cma_id, &conn_param);
> if (ret) {
> printf("udaddy: failure accepting: %d\n", ret);
> @@ -303,19 +296,12 @@ err1:
> return ret;
> }
>
> -static int resolved_handler(struct cmatest_node *node)
> +static int resolved_handler(struct cmatest_node *node,
> + struct rdma_cm_event *event)
> {
> - struct ibv_ah_attr ah_attr;
> - int ret;
> -
> - ret = rdma_get_dst_attr(node->cma_id, test.dst_addr, &ah_attr,
> - &node->remote_qpn, &node->remote_qkey);
> - if (ret) {
> - printf("udaddy: failure getting destination attributes\n");
> - goto err;
> - }
> -
> - node->ah = ibv_create_ah(node->pd, &ah_attr);
> + node->remote_qpn = event->param.ud.qp_num;
> + node->remote_qkey = event->param.ud.qkey;
> + node->ah = ibv_create_ah(node->pd, &event->param.ud.ah_attr);
> if (!node->ah) {
> printf("udaddy: failure creating address handle\n");
> goto err;
> @@ -326,7 +312,7 @@ static int resolved_handler(struct cmate
> return 0;
> err:
> connect_error();
> - return ret;
> + return -1;
> }
>
> static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
> @@ -344,7 +330,7 @@ static int cma_handler(struct rdma_cm_id
> ret = connect_handler(cma_id);
> break;
> case RDMA_CM_EVENT_ESTABLISHED:
> - ret = resolved_handler(cma_id->context);
> + ret = resolved_handler(cma_id->context, event);
> break;
> case RDMA_CM_EVENT_ADDR_ERROR:
> case RDMA_CM_EVENT_ROUTE_ERROR:
> @@ -404,7 +390,7 @@ static int alloc_nodes(void)
>
> for (i = 0; i < connections; i++) {
> test.nodes[i].id = i;
> - if (!is_server) {
> + if (dst_addr) {
> ret = rdma_create_id(test.channel,
> &test.nodes[i].cma_id,
> &test.nodes[i], RDMA_PS_UDP);
> @@ -475,6 +461,28 @@ static int connect_events(void)
> return ret;
> }
>
> +static int get_addr(char *dst, struct sockaddr_in *addr)
> +{
> + struct addrinfo *res;
> + int ret;
> +
> + ret = getaddrinfo(dst, NULL, NULL, &res);
> + if (ret) {
> + printf("getaddrinfo failed - invalid hostname or IP address\n");
> + return ret;
> + }
> +
> + if (res->ai_family != PF_INET) {
> + ret = -1;
> + goto out;
> + }
> +
> + *addr = *(struct sockaddr_in *) res->ai_addr;
> +out:
> + freeaddrinfo(res);
> + return ret;
> +}
> +
> static int run_server(void)
> {
> struct rdma_cm_id *listen_id;
> @@ -487,7 +495,13 @@ static int run_server(void)
> return ret;
> }
>
> - test.src_in.sin_family = PF_INET;
> + if (src_addr) {
> + ret = get_addr(src_addr, &test.src_in);
> + if (ret)
> + goto out;
> + } else
> + test.src_in.sin_family = PF_INET;
> +
> test.src_in.sin_port = 7174;
> ret = rdma_bind_addr(listen_id, test.src_addr);
> if (ret) {
> @@ -526,40 +540,18 @@ out:
> return ret;
> }
>
> -static int get_addr(char *dst, struct sockaddr_in *addr)
> -{
> - struct addrinfo *res;
> - int ret;
> -
> - ret = getaddrinfo(dst, NULL, NULL, &res);
> - if (ret) {
> - printf("getaddrinfo failed - invalid hostname or IP address\n");
> - return ret;
> - }
> -
> - if (res->ai_family != PF_INET) {
> - ret = -1;
> - goto out;
> - }
> -
> - *addr = *(struct sockaddr_in *) res->ai_addr;
> -out:
> - freeaddrinfo(res);
> - return ret;
> -}
> -
> -static int run_client(char *dst, char *src)
> +static int run_client(void)
> {
> int i, ret;
>
> printf("udaddy: starting client\n");
> - if (src) {
> - ret = get_addr(src, &test.src_in);
> + if (src_addr) {
> + ret = get_addr(src_addr, &test.src_in);
> if (ret)
> return ret;
> }
>
> - ret = get_addr(dst, &test.dst_in);
> + ret = get_addr(dst_addr, &test.dst_in);
> if (ret)
> return ret;
>
> @@ -568,7 +560,7 @@ static int run_client(char *dst, char *s
> printf("udaddy: connecting\n");
> for (i = 0; i < connections; i++) {
> ret = rdma_resolve_addr(test.nodes[i].cma_id,
> - src ? test.src_addr : NULL,
> + src_addr ? test.src_addr : NULL,
> test.dst_addr, 2000);
> if (ret) {
> printf("udaddy: failure getting addr: %d\n", ret);
> @@ -601,13 +593,35 @@ out:
>
> int main(int argc, char **argv)
> {
> - int ret;
> + int op, ret;
>
> - if (argc > 3) {
> - printf("usage: %s [server_addr [src_addr]]\n", argv[0]);
> - exit(1);
> + while ((op = getopt(argc, argv, "s:b:c:C:S:")) != -1) {
> + switch (op) {
> + case 's':
> + dst_addr = optarg;
> + break;
> + case 'b':
> + src_addr = optarg;
> + break;
> + case 'c':
> + connections = atoi(optarg);
> + break;
> + case 'C':
> + message_count = atoi(optarg);
> + break;
> + case 'S':
> + message_size = atoi(optarg);
> + break;
> + default:
> + printf("usage: %s\n", argv[0]);
> + printf("\t[-s server_address]\n");
> + printf("\t[-b bind_address]\n");
> + printf("\t[-c connections]\n");
> + printf("\t[-C message_count]\n");
> + printf("\t[-S message_size]\n");
> + exit(1);
> + }
> }
> - is_server = (argc == 1);
>
> test.dst_addr = (struct sockaddr *) &test.dst_in;
> test.src_addr = (struct sockaddr *) &test.src_in;
> @@ -622,10 +636,10 @@ int main(int argc, char **argv)
> if (alloc_nodes())
> exit(1);
>
> - if (is_server)
> - ret = run_server();
> + if (dst_addr)
> + ret = run_client();
> else
> - ret = run_client(argv[1], (argc == 3) ? argv[2] : NULL);
> + ret = run_server();
>
> printf("test complete\n");
> destroy_nodes();
> Index: examples/cmatose.c
> ===================================================================
> --- examples/cmatose.c (revision 9192)
> +++ examples/cmatose.c (working copy)
> @@ -1,5 +1,5 @@
> /*
> - * Copyright (c) 2005 Intel Corporation. All rights reserved.
> + * Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
> *
> * This software is available to you under a choice of one of two
> * licenses. You may choose to be licensed under the terms of the GNU
> @@ -41,6 +41,7 @@
> #include <sys/socket.h>
> #include <netdb.h>
> #include <byteswap.h>
> +#include <getopt.h>
>
> #include <rdma/rdma_cma.h>
>
> @@ -52,12 +53,6 @@ static inline uint64_t cpu_to_be64(uint6
> static inline uint32_t cpu_to_be32(uint32_t x) { return bswap_32(x); }
> #endif
>
> -/*
> - * To execute:
> - * Server: rdma_cmatose
> - * Client: rdma_cmatose <dst_ip>
> - */
> -
> struct cmatest_node {
> int id;
> struct rdma_cm_id *cma_id;
> @@ -85,7 +80,8 @@ static struct cmatest test;
> static int connections = 1;
> static int message_size = 100;
> static int message_count = 10;
> -static int is_server;
> +static char *dst_addr;
> +static char *src_addr;
>
> static int create_message(struct cmatest_node *node)
> {
> @@ -377,7 +373,7 @@ static int alloc_nodes(void)
>
> for (i = 0; i < connections; i++) {
> test.nodes[i].id = i;
> - if (!is_server) {
> + if (dst_addr) {
> ret = rdma_create_id(test.channel,
> &test.nodes[i].cma_id,
> &test.nodes[i], RDMA_PS_TCP);
> @@ -460,6 +456,28 @@ static int disconnect_events(void)
> return ret;
> }
>
> +static int get_addr(char *dst, struct sockaddr_in *addr)
> +{
> + struct addrinfo *res;
> + int ret;
> +
> + ret = getaddrinfo(dst, NULL, NULL, &res);
> + if (ret) {
> + printf("getaddrinfo failed - invalid hostname or IP address\n");
> + return ret;
> + }
> +
> + if (res->ai_family != PF_INET) {
> + ret = -1;
> + goto out;
> + }
> +
> + *addr = *(struct sockaddr_in *) res->ai_addr;
> +out:
> + freeaddrinfo(res);
> + return ret;
> +}
> +
> static int run_server(void)
> {
> struct rdma_cm_id *listen_id;
> @@ -472,12 +490,18 @@ static int run_server(void)
> return ret;
> }
>
> - test.src_in.sin_family = PF_INET;
> + if (src_addr) {
> + ret = get_addr(src_addr, &test.src_in);
> + if (ret)
> + goto out;
> + } else
> + test.src_in.sin_family = PF_INET;
> +
> test.src_in.sin_port = 7471;
> ret = rdma_bind_addr(listen_id, test.src_addr);
> if (ret) {
> printf("cmatose: bind address failed: %d\n", ret);
> - return ret;
> + goto out;
> }
>
> ret = rdma_listen(listen_id, 0);
> @@ -528,40 +552,18 @@ out:
> return ret;
> }
>
> -static int get_addr(char *dst, struct sockaddr_in *addr)
> -{
> - struct addrinfo *res;
> - int ret;
> -
> - ret = getaddrinfo(dst, NULL, NULL, &res);
> - if (ret) {
> - printf("getaddrinfo failed - invalid hostname or IP address\n");
> - return ret;
> - }
> -
> - if (res->ai_family != PF_INET) {
> - ret = -1;
> - goto out;
> - }
> -
> - *addr = *(struct sockaddr_in *) res->ai_addr;
> -out:
> - freeaddrinfo(res);
> - return ret;
> -}
> -
> -static int run_client(char *dst, char *src)
> +static int run_client(void)
> {
> int i, ret, ret2;
>
> printf("cmatose: starting client\n");
> - if (src) {
> - ret = get_addr(src, &test.src_in);
> + if (src_addr) {
> + ret = get_addr(src_addr, &test.src_in);
> if (ret)
> return ret;
> }
>
> - ret = get_addr(dst, &test.dst_in);
> + ret = get_addr(dst_addr, &test.dst_in);
> if (ret)
> return ret;
>
> @@ -570,7 +572,7 @@ static int run_client(char *dst, char *s
> printf("cmatose: connecting\n");
> for (i = 0; i < connections; i++) {
> ret = rdma_resolve_addr(test.nodes[i].cma_id,
> - src ? test.src_addr : NULL,
> + src_addr ? test.src_addr : NULL,
> test.dst_addr, 2000);
> if (ret) {
> printf("cmatose: failure getting addr: %d\n", ret);
> @@ -597,7 +599,6 @@ static int run_client(char *dst, char *s
> }
>
> printf("data transfers complete\n");
> -
> }
>
> ret = 0;
> @@ -611,13 +612,35 @@ out:
>
> int main(int argc, char **argv)
> {
> - int ret;
> + int op, ret;
>
> - if (argc > 3) {
> - printf("usage: %s [server_addr [src_addr]]\n", argv[0]);
> - exit(1);
> + while ((op = getopt(argc, argv, "s:b:c:C:S:")) != -1) {
> + switch (op) {
> + case 's':
> + dst_addr = optarg;
> + break;
> + case 'b':
> + src_addr = optarg;
> + break;
> + case 'c':
> + connections = atoi(optarg);
> + break;
> + case 'C':
> + message_count = atoi(optarg);
> + break;
> + case 'S':
> + message_size = atoi(optarg);
> + break;
> + default:
> + printf("usage: %s\n", argv[0]);
> + printf("\t[-s server_address]\n");
> + printf("\t[-b bind_address]\n");
> + printf("\t[-c connections]\n");
> + printf("\t[-C message_count]\n");
> + printf("\t[-S message_size]\n");
> + exit(1);
> + }
> }
> - is_server = (argc == 1);
>
> test.dst_addr = (struct sockaddr *) &test.dst_in;
> test.src_addr = (struct sockaddr *) &test.src_in;
> @@ -633,10 +656,10 @@ int main(int argc, char **argv)
> if (alloc_nodes())
> exit(1);
>
> - if (is_server)
> - ret = run_server();
> + if (dst_addr)
> + ret = run_client();
> else
> - ret = run_client(argv[1], (argc == 3) ? argv[2] : NULL);
> + ret = run_server();
>
> printf("test complete\n");
> destroy_nodes();
>
>
> _______________________________________________
> openib-general mailing list
> openib-general at openib.org
> http://openib.org/mailman/listinfo/openib-general
>
> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
>
More information about the general
mailing list