[openib-general] [PATCH] librdmacm: updated librdmacm to work with proposed 2.6.20 kernel CMA

Steve Wise swise at opengridcomputing.com
Wed Nov 1 14:13:56 PST 2006


Sean,

This patch removes rdma_get/set_option().  Is that what you intended?



On Wed, 2006-10-25 at 13:49 -0700, Sean Hefty wrote:
> Updates the librdmacm to work with ABI version 3, which is the proposed
> kernel changes for inclusion in 2.6.20.
> 
> Test programs are also updated.
> 
> Signed-off-by: Sean Hefty <sean.hefty at intel.com>
> ---
> Index: include/rdma/rdma_cma_abi.h
> ===================================================================
> --- include/rdma/rdma_cma_abi.h	(revision 9192)
> +++ include/rdma/rdma_cma_abi.h	(working copy)
> @@ -33,14 +33,15 @@
>  #ifndef RDMA_CMA_ABI_H
>  #define RDMA_CMA_ABI_H
>  
> +#include <infiniband/kern-abi.h>
>  #include <infiniband/sa-kern-abi.h>
>  
>  /*
>   * This file must be kept in sync with the kernel's version of rdma_user_cm.h
>   */
>  
> -#define RDMA_USER_CM_MIN_ABI_VERSION	1
> -#define RDMA_USER_CM_MAX_ABI_VERSION	2
> +#define RDMA_USER_CM_MIN_ABI_VERSION	3
> +#define RDMA_USER_CM_MAX_ABI_VERSION	3
>  
>  #define RDMA_MAX_PRIVATE_DATA		256
>  
> @@ -60,7 +61,7 @@ enum {
>  	UCMA_CMD_GET_EVENT,
>  	UCMA_CMD_GET_OPTION,
>  	UCMA_CMD_SET_OPTION,
> -	UCMA_CMD_GET_DST_ATTR,
> +	UCMA_CMD_ESTABLISH,
>  	UCMA_CMD_JOIN_MCAST,
>  	UCMA_CMD_LEAVE_MCAST
>  };
> @@ -71,11 +72,6 @@ struct ucma_abi_cmd_hdr {
>  	__u16 out;
>  };
>  
> -struct ucma_abi_create_id_v1 {
> -	__u64 uid;
> -	__u64 response;
> -};
> -
>  struct ucma_abi_create_id {
>  	__u64 uid;
>  	__u64 response;
> @@ -133,7 +129,7 @@ struct ucma_abi_query_route_resp {
>  
>  struct ucma_abi_conn_param {
>  	__u32 qp_num;
> -	__u32 qp_type;
> +	__u32 reserved;
>  	__u8  private_data[RDMA_MAX_PRIVATE_DATA];
>  	__u8  private_data_len;
>  	__u8  srq;
> @@ -145,6 +141,15 @@ struct ucma_abi_conn_param {
>  	__u8  valid;
>  };
>  
> +struct ucma_abi_ud_param {
> +	__u32 qp_num;
> +	__u32 qkey;
> +	struct ibv_kern_ah_attr ah_attr;
> +	__u8 private_data[RDMA_MAX_PRIVATE_DATA];
> +	__u8 private_data_len;
> +	__u8 reserved[7];
> +};
> +
>  struct ucma_abi_connect {
>  	struct ucma_abi_conn_param conn_param;
>  	__u32 id;
> @@ -180,25 +185,13 @@ struct ucma_abi_init_qp_attr {
>  	__u32 qp_state;
>  };
>  
> -struct ucma_abi_join_mcast {
> -	__u32 id;
> -	struct sockaddr_in6 addr;
> -	__u64 uid;
> -};
> -
> -struct ucma_abi_leave_mcast {
> +struct ucma_abi_establish {
>  	__u32 id;
> -	struct sockaddr_in6 addr;
> -};
> -
> -struct ucma_abi_dst_attr_resp {
> -	__u32 remote_qpn;
> -	__u32 remote_qkey;
> -	struct ibv_kern_ah_attr ah_attr;
>  };
>  
> -struct ucma_abi_get_dst_attr {
> -	__u64 response;
> +struct ucma_abi_join_mcast {
> +	__u64 response;		/* ucma_abi_create_id_resp */
> +	__u64 uid;
>  	struct sockaddr_in6 addr;
>  	__u32 id;
>  };
> @@ -212,30 +205,10 @@ struct ucma_abi_event_resp {
>  	__u32 id;
>  	__u32 event;
>  	__u32 status;
> -	__u8  private_data_len;
> -	__u8  reserved[3];
> -	__u8  private_data[RDMA_MAX_PRIVATE_DATA];
> -};
> -
> -struct ucma_abi_get_option {
> -	__u64 response;
> -	__u64 optval;
> -	__u32 id;
> -	__u32 level;
> -	__u32 optname;
> -	__u32 optlen;
> -};
> -
> -struct ucma_abi_get_option_resp {
> -	__u32 optlen;
> -};
> -
> -struct ucma_abi_set_option {
> -	__u64 optval;
> -	__u32 id;
> -	__u32 level;
> -	__u32 optname;
> -	__u32 optlen;
> +	union {
> +		struct ucma_abi_conn_param conn;
> +		struct ucma_abi_ud_param   ud;
> +	} param;
>  };
>  
>  #endif /* RDMA_CMA_ABI_H */
> Index: include/rdma/rdma_cma.h
> ===================================================================
> --- include/rdma/rdma_cma.h	(revision 9272)
> +++ include/rdma/rdma_cma.h	(working copy)
> @@ -61,11 +61,11 @@ enum rdma_port_space {
>  	RDMA_PS_UDP  = 0x0111,
>  };
>  
> -/* Protocol levels for get/set options. */
> -enum {
> -	RDMA_PROTO_IP = 0,
> -	RDMA_PROTO_IB = 1,
> -};
> +/*
> + * Global qkey value for all UD QPs and multicast groups created via the 
> + * RDMA CM.
> + */
> +#define RDMA_UD_QKEY 0x01234567
>  
>  struct ib_addr {
>  	union ibv_gid	sgid;
> @@ -74,8 +74,12 @@ struct ib_addr {
>  };
>  
>  struct rdma_addr {
> -	struct sockaddr_in6	src_addr;
> -	struct sockaddr_in6	dst_addr;
> +	struct sockaddr		src_addr;
> +	uint8_t			src_pad[sizeof(struct sockaddr_storage) -
> +					sizeof(struct sockaddr)];
> +	struct sockaddr		dst_addr;
> +	uint8_t			dst_pad[sizeof(struct sockaddr_storage) -
> +					sizeof(struct sockaddr)];
>  	union {
>  		struct ib_addr	ibaddr;
>  	} addr;
> @@ -101,11 +105,25 @@ struct rdma_cm_id {
>  	uint8_t			 port_num;
>  };
>  
> -struct rdma_multicast_data {
> -	void		*context;
> -	struct sockaddr addr;
> -	uint8_t		pad[sizeof(struct sockaddr_in6) -
> -			    sizeof(struct sockaddr)];
> +struct rdma_conn_param {
> +	const void *private_data;
> +	uint8_t private_data_len;
> +	uint8_t responder_resources;
> +	uint8_t initiator_depth;
> +	uint8_t flow_control;
> +	uint8_t retry_count;		/* ignored when accepting */
> +	uint8_t rnr_retry_count;
> +	/* Fields below ignored if a QP is created on the rdma_cm_id. */
> +	uint8_t srq;
> +	uint32_t qp_num;
> +};
> +
> +struct rdma_ud_param {
> +	const void *private_data;
> +	uint8_t private_data_len;
> +	struct ibv_ah_attr ah_attr;
> +	uint32_t qp_num;
> +	uint32_t qkey;
>  };
>  
>  struct rdma_cm_event {
> @@ -113,8 +131,10 @@ struct rdma_cm_event {
>  	struct rdma_cm_id	*listen_id;
>  	enum rdma_cm_event_type	 event;
>  	int			 status;
> -	void			*private_data;
> -	uint8_t			 private_data_len;
> +	union {
> +		struct rdma_conn_param conn;
> +		struct rdma_ud_param   ud;
> +	} param;
>  };
>  
>  /**
> @@ -206,20 +226,6 @@ int rdma_create_qp(struct rdma_cm_id *id
>   */
>  void rdma_destroy_qp(struct rdma_cm_id *id);
>  
> -struct rdma_conn_param {
> -	const void *private_data;
> -	uint8_t private_data_len;
> -	uint8_t responder_resources;
> -	uint8_t initiator_depth;
> -	uint8_t flow_control;
> -	uint8_t retry_count;		/* ignored when accepting */
> -	uint8_t rnr_retry_count;
> -	/* Fields below ignored if a QP is created on the rdma_cm_id. */
> -	uint8_t srq;
> -	uint32_t qp_num;
> -	enum ibv_qp_type qp_type;
> -};
> -
>  /**
>   * rdma_connect - Initiate an active connection request.
>   *
> @@ -251,6 +257,16 @@ int rdma_reject(struct rdma_cm_id *id, c
>  		uint8_t private_data_len);
>  
>  /**
> + * rdma_establish - Forces a connection state to established.
> + * @id: Connection identifier to transition to established.
> + *
> + * This routine should be invoked by users who receive messages on a
> + * QP before being notified that the connection has been established by the
> + * RDMA CM.
> + */
> +int rdma_establish(struct rdma_cm_id *id);
> +
> +/**
>   * rdma_disconnect - This function disconnects the associated QP and
>   *   transitions it into the error state.
>   */
> @@ -298,40 +314,17 @@ int rdma_get_cm_event(struct rdma_event_
>   */
>  int rdma_ack_cm_event(struct rdma_cm_event *event);
>  
> -/**
> - * rdma_get_option - Retrieve options for an rdma_cm_id.
> - * @id: Communication identifier to retrieve option for.
> - * @level: Protocol level of the option to retrieve.
> - * @optname: Name of the option to retrieve.
> - * @optval: Buffer to receive the returned options.
> - * @optlen: On input, the size of the %optval buffer.  On output, the
> - *   size of the returned data.
> - */
> -int rdma_get_option(struct rdma_cm_id *id, int level, int optname,
> -		    void *optval, size_t *optlen);
> -
> -/**
> - * rdma_set_option - Set options for an rdma_cm_id.
> - * @id: Communication identifier to set option for.
> - * @level: Protocol level of the option to set.
> - * @optname: Name of the option to set.
> - * @optval: Reference to the option data.
> - * @optlen: The size of the %optval buffer.
> - */
> -int rdma_set_option(struct rdma_cm_id *id, int level, int optname,
> -		    void *optval, size_t optlen);
> -
>  static inline uint16_t rdma_get_src_port(struct rdma_cm_id *id)
>  {
> -	return	id->route.addr.src_addr.sin6_family == PF_INET6 ?
> -		id->route.addr.src_addr.sin6_port :
> +	return	id->route.addr.src_addr.sa_family == PF_INET6 ?
> +		((struct sockaddr_in6 *) &id->route.addr.src_addr)->sin6_port :
>  		((struct sockaddr_in *) &id->route.addr.src_addr)->sin_port;
>  }
>  
>  static inline uint16_t rdma_get_dst_port(struct rdma_cm_id *id)
>  {
> -	return	id->route.addr.dst_addr.sin6_family == PF_INET6 ?
> -		id->route.addr.dst_addr.sin6_port :
> +	return	id->route.addr.dst_addr.sa_family == PF_INET6 ?
> +		((struct sockaddr_in6 *) &id->route.addr.dst_addr)->sin6_port :
>  		((struct sockaddr_in *) &id->route.addr.dst_addr)->sin_port;
>  }
>  
> Index: src/cma.c
> ===================================================================
> --- src/cma.c	(revision 9696)
> +++ src/cma.c	(working copy)
> @@ -1,5 +1,5 @@
>  /*
> - * Copyright (c) 2005 Intel Corporation.  All rights reserved.
> + * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
>   *
>   * This software is available to you under a choice of one of two
>   * licenses.  You may choose to be licensed under the terms of the GNU
> @@ -54,7 +54,6 @@
>  #include <infiniband/marshall.h>
>  #include <rdma/rdma_cma.h>
>  #include <rdma/rdma_cma_abi.h>
> -#include <rdma/rdma_cma_ib.h>
>  
>  #define PFX "librdmacm: "
>  
> @@ -116,6 +115,28 @@ struct cma_id_private {
>  	pthread_cond_t	  cond;
>  	pthread_mutex_t	  mut;
>  	uint32_t	  handle;
> +	struct cma_multicast *mc_list;
> +};
> +
> +struct cma_multicast {
> +	struct cma_multicast  *next;
> +	struct cma_id_private *id_priv;
> +	void		*context;
> +	int		events_completed;
> +	pthread_cond_t	cond;
> +	uint32_t	handle;
> +	union ibv_gid	mgid;
> +	uint16_t	mlid;
> +	struct sockaddr addr;
> +	uint8_t		pad[sizeof(struct sockaddr_in6) -
> +			    sizeof(struct sockaddr)];
> +};
> +
> +struct cma_event {
> +	struct rdma_cm_event	event;
> +	uint8_t			private_data[RDMA_MAX_PRIVATE_DATA];
> +	struct cma_id_private	*id_priv;
> +	struct cma_multicast	*mc;
>  };
>  
>  static struct cma_device *cma_dev_array;
> @@ -335,41 +356,6 @@ err:	ucma_free_id(id_priv);
>  	return NULL;
>  }
>  
> -static int ucma_create_id_v1(struct rdma_event_channel *channel,
> -			     struct rdma_cm_id **id, void *context,
> -			     enum rdma_port_space ps)
> -{
> -	struct ucma_abi_create_id_resp *resp;
> -	struct ucma_abi_create_id_v1 *cmd;
> -	struct cma_id_private *id_priv;
> -	void *msg;
> -	int ret, size;
> -
> -	if (ps != RDMA_PS_TCP) {
> -		fprintf(stderr, "librdmacm: Kernel ABI does not support "
> -				"requested port space.\n");
> -		return -EPROTONOSUPPORT;
> -	}
> -
> -	id_priv = ucma_alloc_id(channel, context, ps);
> -	if (!id_priv)
> -		return -ENOMEM;
> -
> -	CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_CREATE_ID, size);
> -	cmd->uid = (uintptr_t) id_priv;
> -
> -	ret = write(channel->fd, msg, size);
> -	if (ret != size)
> -		goto err;
> -
> -	id_priv->handle = resp->id;
> -	*id = &id_priv->id;
> -	return 0;
> -
> -err:	ucma_free_id(id_priv);
> -	return ret;
> -}
> -
>  int rdma_create_id(struct rdma_event_channel *channel,
>  		   struct rdma_cm_id **id, void *context,
>  		   enum rdma_port_space ps)
> @@ -384,9 +370,6 @@ int rdma_create_id(struct rdma_event_cha
>  	if (ret)
>  		return ret;
>  
> -	if (abi_ver == 1)
> -		return ucma_create_id_v1(channel, id, context, ps);
> -
>  	id_priv = ucma_alloc_id(channel, context, ps);
>  	if (!id_priv)
>  		return -ENOMEM;
> @@ -492,9 +475,9 @@ static int ucma_query_route(struct rdma_
>  	       sizeof id->route.addr.addr.ibaddr.dgid);
>  	id->route.addr.addr.ibaddr.pkey = resp->ib_route[0].pkey;
>  	memcpy(&id->route.addr.src_addr, &resp->src_addr,
> -	       sizeof id->route.addr.src_addr);
> +	       sizeof resp->src_addr);
>  	memcpy(&id->route.addr.dst_addr, &resp->dst_addr,
> -	       sizeof id->route.addr.dst_addr);
> +	       sizeof resp->dst_addr);
>  
>  	if (!id_priv->cma_dev && resp->node_guid) {
>  		ret = ucma_get_device(id_priv, resp->node_guid);
> @@ -696,7 +679,7 @@ static int ucma_init_ib_qp(struct cma_id
>  
>  	qp_attr.port_num = id_priv->id.port_num;
>  	qp_attr.qp_state = IBV_QPS_INIT;
> -	qp_attr.qp_access_flags = IBV_ACCESS_LOCAL_WRITE;
> +	qp_attr.qp_access_flags = 0;
>  	return ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_ACCESS_FLAGS |
>  					   IBV_QP_PKEY_INDEX | IBV_QP_PORT);
>  }
> @@ -767,11 +750,9 @@ void rdma_destroy_qp(struct rdma_cm_id *
>  
>  static void ucma_copy_conn_param_to_kern(struct ucma_abi_conn_param *dst,
>  					 struct rdma_conn_param *src,
> -					 uint32_t qp_num,
> -					 enum ibv_qp_type qp_type, uint8_t srq)
> +					 uint32_t qp_num, uint8_t srq)
>  {
>  	dst->qp_num = qp_num;
> -	dst->qp_type = qp_type;
>  	dst->srq = srq;
>  	dst->responder_resources = src->responder_resources;
>  	dst->initiator_depth = src->initiator_depth;
> @@ -799,12 +780,11 @@ int rdma_connect(struct rdma_cm_id *id, 
>  	cmd->id = id_priv->handle;
>  	if (id->qp)
>  		ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
> -					     id->qp->qp_num, id->qp->qp_type,
> +					     id->qp->qp_num,
>  					     (id->qp->srq != NULL));
>  	else
>  		ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
>  					     conn_param->qp_num,
> -					     conn_param->qp_type,
>  					     conn_param->srq);
>  
>  	ret = write(id->channel->fd, msg, size);
> @@ -852,12 +832,11 @@ int rdma_accept(struct rdma_cm_id *id, s
>  	cmd->uid = (uintptr_t) id_priv;
>  	if (id->qp)
>  		ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
> -					     id->qp->qp_num, id->qp->qp_type,
> +					     id->qp->qp_num,
>  					     (id->qp->srq != NULL));
>  	else
>  		ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param,
>  					     conn_param->qp_num,
> -					     conn_param->qp_type,
>  					     conn_param->srq);
>  
>  	ret = write(id->channel->fd, msg, size);
> @@ -894,6 +873,24 @@ int rdma_reject(struct rdma_cm_id *id, c
>  	return 0;
>  }
>  
> +int rdma_establish(struct rdma_cm_id *id)
> +{
> +	struct ucma_abi_establish *cmd;
> +	struct cma_id_private *id_priv;
> +	void *msg;
> +	int ret, size;
> +	
> +	CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_ESTABLISH, size);
> +
> +	id_priv = container_of(id, struct cma_id_private, id);
> +	cmd->id = id_priv->handle;
> +	ret = write(id->channel->fd, msg, size);
> +	if (ret != size)
> +		return (ret > 0) ? -ENODATA : ret;
> +
> +	return 0;
> +}
> +
>  int rdma_disconnect(struct rdma_cm_id *id)
>  {
>  	struct ucma_abi_disconnect *cmd;
> @@ -929,74 +926,102 @@ int rdma_join_multicast(struct rdma_cm_i
>  			void *context)
>  {
>  	struct ucma_abi_join_mcast *cmd;
> +	struct ucma_abi_create_id_resp *resp;
>  	struct cma_id_private *id_priv;
> +	struct cma_multicast *mc, **pos;
>  	void *msg;
>  	int ret, size, addrlen;
>  	
> +	id_priv = container_of(id, struct cma_id_private, id);
>  	addrlen = ucma_addrlen(addr);
>  	if (!addrlen)
>  		return -EINVAL;
>  
> -	CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_JOIN_MCAST, size);
> -	id_priv = container_of(id, struct cma_id_private, id);
> +	mc = malloc(sizeof *mc);
> +	if (!mc)
> +		return -ENOMEM;
> +
> +	memset(mc, 0, sizeof *mc);
> +	mc->context = context;
> +	mc->id_priv = id_priv;
> +	memcpy(&mc->addr, addr, addrlen);
> +	if (pthread_cond_init(&id_priv->cond, NULL)) {
> +		ret = -1;
> +		goto err1;
> +	}
> +
> +	pthread_mutex_lock(&id_priv->mut);
> +	mc->next = id_priv->mc_list;
> +	id_priv->mc_list = mc;
> +	pthread_mutex_unlock(&id_priv->mut);
> +
> +	CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_JOIN_MCAST, size);
>  	cmd->id = id_priv->handle;
>  	memcpy(&cmd->addr, addr, addrlen);
> -	cmd->uid = (uintptr_t) context;
> +	cmd->uid = (uintptr_t) mc;
>  
>  	ret = write(id->channel->fd, msg, size);
> -	if (ret != size)
> -		return (ret > 0) ? -ENODATA : ret;
> +	if (ret != size) {
> +		ret = (ret > 0) ? -ENODATA : ret;
> +		goto err2;
> +	}
>  
> +	mc->handle = resp->id;
>  	return 0;
> +err2:
> +	pthread_mutex_lock(&id_priv->mut);
> +	for (pos = &id_priv->mc_list; *pos != mc; pos = &(*pos)->next)
> +		;
> +	*pos = mc->next;
> +	pthread_mutex_unlock(&id_priv->mut);
> +err1:
> +	free(mc);
> +	return ret;
>  }
>  
>  int rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
>  {
> -	struct ucma_abi_leave_mcast *cmd;
> +	struct ucma_abi_destroy_id *cmd;
> +	struct ucma_abi_destroy_id_resp *resp;
>  	struct cma_id_private *id_priv;
> +	struct cma_multicast *mc, **pos;
>  	void *msg;
>  	int ret, size, addrlen;
> -	struct ibv_ah_attr ah_attr;
> -	uint32_t qp_info;
>  	
>  	addrlen = ucma_addrlen(addr);
>  	if (!addrlen)
>  		return -EINVAL;
>  
> -	CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_LEAVE_MCAST, size);
>  	id_priv = container_of(id, struct cma_id_private, id);
> -	cmd->id = id_priv->handle;
> -	memcpy(&cmd->addr, addr, addrlen);
> +	pthread_mutex_lock(&id_priv->mut);
> +	for (pos = &id_priv->mc_list; *pos; pos = &(*pos)->next)
> +		if (!memcmp(&(*pos)->addr, addr, addrlen))
> +			break;
>  
> -	if (id->qp) {
> -		ret = rdma_get_dst_attr(id, addr, &ah_attr, &qp_info, &qp_info);
> -		if (ret)
> -			goto out;
> +	mc = *pos;
> +	if (*pos)
> +		*pos = mc->next;
> +	pthread_mutex_unlock(&id_priv->mut);
> +	if (!mc)
> +		return -EADDRNOTAVAIL;
>  
> -		ret = ibv_detach_mcast(id->qp, &ah_attr.grh.dgid, ah_attr.dlid);
> -		if (ret)
> -			goto out;
> -	}
> +	if (id->qp)
> +		ibv_detach_mcast(id->qp, &mc->mgid, mc->mlid);
>  	
> +	CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_LEAVE_MCAST, size);
> +	cmd->id = mc->handle;
> +
>  	ret = write(id->channel->fd, msg, size);
>  	if (ret != size)
>  		ret = (ret > 0) ? -ENODATA : ret;
> -out:
> -	return ret;
> -}
>  
> -static void ucma_copy_event_from_kern(struct rdma_cm_event *dst,
> -				      struct ucma_abi_event_resp *src)
> -{
> -	dst->event = src->event;
> -	dst->status = src->status;
> -	dst->private_data_len = src->private_data_len;
> -	if (src->private_data_len) {
> -		dst->private_data = dst + 1;
> -		memcpy(dst->private_data, src->private_data,
> -		       src->private_data_len);
> -	} else
> -		dst->private_data = NULL;
> +	pthread_mutex_lock(&id_priv->mut);
> +	while (mc->events_completed < resp->events_reported)
> +		pthread_cond_wait(&mc->cond, &id_priv->mut);
> +	pthread_mutex_unlock(&id_priv->mut);
> +
> +	free(mc);
> +	return ret;
>  }
>  
>  static void ucma_complete_event(struct cma_id_private *id_priv)
> @@ -1007,38 +1032,49 @@ static void ucma_complete_event(struct c
>  	pthread_mutex_unlock(&id_priv->mut);
>  }
>  
> +static void ucma_complete_mc_event(struct cma_multicast *mc)
> +{
> +	pthread_mutex_lock(&mc->id_priv->mut);
> +	mc->events_completed++;
> +	pthread_cond_signal(&mc->cond);
> +	mc->id_priv->events_completed++;
> +	pthread_cond_signal(&mc->id_priv->cond);
> +	pthread_mutex_unlock(&mc->id_priv->mut);
> +}
> +
>  int rdma_ack_cm_event(struct rdma_cm_event *event)
>  {
> -	struct rdma_cm_id *id;
> +	struct cma_event *evt;
>  
>  	if (!event)
>  		return -EINVAL;
>  
> -	id = (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) ?
> -	     event->listen_id : event->id;
> +	evt = container_of(event, struct cma_event, event);
>  
> -	ucma_complete_event(container_of(id, struct cma_id_private, id));
> -	free(event);
> +	if (evt->mc)
> +		ucma_complete_mc_event(evt->mc);
> +	else
> +		ucma_complete_event(evt->id_priv);
> +	free(evt);
>  	return 0;
>  }
>  
> -static int ucma_process_conn_req(struct rdma_cm_event *event,
> +static int ucma_process_conn_req(struct cma_event *evt,
>  				 uint32_t handle)
>  {
> -	struct cma_id_private *listen_id_priv, *id_priv;
> +	struct cma_id_private *id_priv;
>  	int ret;
>  
> -	listen_id_priv = container_of(event->id, struct cma_id_private, id);
> -	id_priv = ucma_alloc_id(event->id->channel, event->id->context,
> -				event->id->ps);
> +	id_priv = ucma_alloc_id(evt->id_priv->id.channel,
> +				evt->id_priv->id.context, evt->id_priv->id.ps);
>  	if (!id_priv) {
> -		ucma_destroy_kern_id(event->id->channel->fd, handle);
> +		ucma_destroy_kern_id(evt->id_priv->id.channel->fd, handle);
>  		ret = -ENOMEM;
>  		goto err;
>  	}
>  
> -	event->listen_id = event->id;
> -	event->id = &id_priv->id;
> +	evt->event.listen_id = &evt->id_priv->id;
> +	evt->event.id = &id_priv->id;
>  	id_priv->handle = handle;
>  
>  	ret = ucma_query_route(&id_priv->id);
> @@ -1049,7 +1085,7 @@ static int ucma_process_conn_req(struct 
>  
>  	return 0;
>  err:
> -	ucma_complete_event(listen_id_priv);
> +	ucma_complete_event(evt->id_priv);
>  	return ret;
>  }
>  
> @@ -1093,34 +1129,54 @@ static int ucma_process_establish(struct
>  	return ret;
>  }
>  
> -static void ucma_process_mcast(struct rdma_cm_id *id, struct rdma_cm_event *evt)
> +static int ucma_process_join(struct cma_event *evt)
>  {
> -	struct ucma_abi_join_mcast kmc_data;
> -	struct rdma_multicast_data *mc_data;
> -	struct ibv_ah_attr ah_attr;
> -	uint32_t qp_info;
> -
> -	kmc_data = *(struct ucma_abi_join_mcast *) evt->private_data;
> -
> -	mc_data = evt->private_data;
> -	mc_data->context = (void *) (uintptr_t) kmc_data.uid;
> -	memcpy(&mc_data->addr, &kmc_data.addr,
> -	       ucma_addrlen((struct sockaddr *) &kmc_data.addr));
> -
> -	if (evt->status || !id->qp)
> -		return;
> -
> -	evt->status = rdma_get_dst_attr(id, &mc_data->addr, &ah_attr,
> -					&qp_info, &qp_info);
> -	if (evt->status)
> -		goto err;
> +	evt->mc->mgid = evt->event.param.ud.ah_attr.grh.dgid;
> +	evt->mc->mlid = evt->event.param.ud.ah_attr.dlid;
>  
> -	evt->status = ibv_attach_mcast(id->qp, &ah_attr.grh.dgid, ah_attr.dlid);
> -	if (evt->status)
> -		goto err;
> -	return;
> -err:
> -	evt->event = RDMA_CM_EVENT_MULTICAST_ERROR;
> +	if (evt->id_priv->id.qp)
> +		return ibv_attach_mcast(evt->id_priv->id.qp,
> +					&evt->mc->mgid, evt->mc->mlid);
> +	else
> +		return 0;
> +}
> +
> +static void ucma_copy_conn_event(struct cma_event *event,
> +				 struct ucma_abi_conn_param *src)
> +{
> +	struct rdma_conn_param *dst = &event->event.param.conn;
> +
> +	dst->private_data_len = src->private_data_len;
> +	if (src->private_data_len) {
> +		dst->private_data = &event->private_data;
> +		memcpy(&event->private_data, src->private_data,
> +		       src->private_data_len);
> +	}
> +
> +	dst->responder_resources = src->responder_resources;
> +	dst->initiator_depth = src->initiator_depth;
> +	dst->flow_control = src->flow_control;
> +	dst->retry_count = src->retry_count;
> +	dst->rnr_retry_count = src->rnr_retry_count;
> +	dst->srq = src->srq;
> +	dst->qp_num = src->qp_num;
> +}
> +
> +static void ucma_copy_ud_event(struct cma_event *event,
> +			       struct ucma_abi_ud_param *src)
> +{
> +	struct rdma_ud_param *dst = &event->event.param.ud;
> +
> +	dst->private_data_len = src->private_data_len;
> +	if (src->private_data_len) {
> +		dst->private_data = &event->private_data;
> +		memcpy(&event->private_data, src->private_data,
> +		       src->private_data_len);
> +	}
> +
> +	ibv_copy_ah_attr_from_kern(&dst->ah_attr, &src->ah_attr);
> +	dst->qp_num = src->qp_num;
> +	dst->qkey = src->qkey;
>  }
>  
>  int rdma_get_cm_event(struct rdma_event_channel *channel,
> @@ -1128,8 +1184,7 @@ int rdma_get_cm_event(struct rdma_event_
>  {
>  	struct ucma_abi_event_resp *resp;
>  	struct ucma_abi_get_event *cmd;
> -	struct cma_id_private *id_priv;
> -	struct rdma_cm_event *evt;
> +	struct cma_event *evt;
>  	void *msg;
>  	int ret, size;
>  
> @@ -1140,155 +1195,119 @@ int rdma_get_cm_event(struct rdma_event_
>  	if (!event)
>  		return -EINVAL;
>  
> -	evt = malloc(sizeof *evt + RDMA_MAX_PRIVATE_DATA);
> +	evt = malloc(sizeof *evt);
>  	if (!evt)
>  		return -ENOMEM;
>  
>  retry:
> +	memset(evt, 0, sizeof *evt);
>  	CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_GET_EVENT, size);
>  	ret = write(channel->fd, msg, size);
>  	if (ret != size) {
>  		free(evt);
>  		return (ret > 0) ? -ENODATA : ret;
>  	}
> -
> -	id_priv = (void *) (uintptr_t) resp->uid;
> -	evt->id = &id_priv->id;
> -	ucma_copy_event_from_kern(evt, resp);
>  	
> -	switch (evt->event) {
> +	evt->event.event = resp->event;
> +	switch (resp->event) {
>  	case RDMA_CM_EVENT_ADDR_RESOLVED:
> -		evt->status = ucma_query_route(&id_priv->id);
> -		if (evt->status)
> -			evt->event = RDMA_CM_EVENT_ADDR_ERROR;
> +		evt->id_priv = (void *) (uintptr_t) resp->uid;
> +		evt->event.id = &evt->id_priv->id;
> +		evt->event.status = ucma_query_route(&evt->id_priv->id);
> +		if (evt->event.status)
> +			evt->event.event = RDMA_CM_EVENT_ADDR_ERROR;
>  		break;
>  	case RDMA_CM_EVENT_ROUTE_RESOLVED:
> -		evt->status = ucma_query_route(&id_priv->id);
> -		if (evt->status)
> -			evt->event = RDMA_CM_EVENT_ROUTE_ERROR;
> +		evt->id_priv = (void *) (uintptr_t) resp->uid;
> +		evt->event.id = &evt->id_priv->id;
> +		evt->event.status = ucma_query_route(&evt->id_priv->id);
> +		if (evt->event.status)
> +			evt->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
>  		break;
>  	case RDMA_CM_EVENT_CONNECT_REQUEST:
> +		evt->id_priv = (void *) (uintptr_t) resp->uid;
> +		if (evt->id_priv->id.ps == RDMA_PS_TCP)
> +			ucma_copy_conn_event(evt, &resp->param.conn);
> +		else
> +			ucma_copy_ud_event(evt, &resp->param.ud);
> +
>  		ret = ucma_process_conn_req(evt, resp->id);
>  		if (ret)
>  			goto retry;
>  		break;
>  	case RDMA_CM_EVENT_CONNECT_RESPONSE:
> -		evt->status = ucma_process_conn_resp(id_priv);
> -		if (!evt->status)
> -			evt->event = RDMA_CM_EVENT_ESTABLISHED;
> +		evt->id_priv = (void *) (uintptr_t) resp->uid;
> +		evt->event.id = &evt->id_priv->id;
> +		ucma_copy_conn_event(evt, &resp->param.conn);
> +		evt->event.status = ucma_process_conn_resp(evt->id_priv);
> +		if (!evt->event.status)
> +			evt->event.event = RDMA_CM_EVENT_ESTABLISHED;
>  		else {
> -			evt->event = RDMA_CM_EVENT_CONNECT_ERROR;
> -			id_priv->connect_error = 1;
> +			evt->event.event = RDMA_CM_EVENT_CONNECT_ERROR;
> +			evt->id_priv->connect_error = 1;
>  		}
>  		break;
>  	case RDMA_CM_EVENT_ESTABLISHED:
> -		if (id_priv->id.ps == RDMA_PS_UDP)
> +		evt->id_priv = (void *) (uintptr_t) resp->uid;
> +		evt->event.id = &evt->id_priv->id;
> +		if (evt->id_priv->id.ps == RDMA_PS_UDP) {
> +			ucma_copy_ud_event(evt, &resp->param.ud);
>  			break;
> +		}
>  
> -		evt->status = ucma_process_establish(&id_priv->id);
> -		if (evt->status) {
> -			evt->event = RDMA_CM_EVENT_CONNECT_ERROR;
> -			id_priv->connect_error = 1;
> +		ucma_copy_conn_event(evt, &resp->param.conn);
> +		evt->event.status = ucma_process_establish(&evt->id_priv->id);
> +		if (evt->event.status) {
> +			evt->event.event = RDMA_CM_EVENT_CONNECT_ERROR;
> +			evt->id_priv->connect_error = 1;
>  		}
>  		break;
>  	case RDMA_CM_EVENT_REJECTED:
> -		if (id_priv->connect_error) {
> -			ucma_complete_event(id_priv);
> +		evt->id_priv = (void *) (uintptr_t) resp->uid;
> +		if (evt->id_priv->connect_error) {
> +			ucma_complete_event(evt->id_priv);
>  			goto retry;
>  		}
> -		ucma_modify_qp_err(evt->id);
> +		evt->event.id = &evt->id_priv->id;
> +		ucma_copy_conn_event(evt, &resp->param.conn);
> +		ucma_modify_qp_err(evt->event.id);
>  		break;
>  	case RDMA_CM_EVENT_DISCONNECTED:
> -		if (id_priv->connect_error) {
> -			ucma_complete_event(id_priv);
> +		evt->id_priv = (void *) (uintptr_t) resp->uid;
> +		if (evt->id_priv->connect_error) {
> +			ucma_complete_event(evt->id_priv);
>  			goto retry;
>  		}
> +		evt->event.id = &evt->id_priv->id;
> +		ucma_copy_conn_event(evt, &resp->param.conn);
>  		break;
>  	case RDMA_CM_EVENT_MULTICAST_JOIN:
> +		evt->mc = (void *) (uintptr_t) resp->uid;
> +		evt->id_priv = evt->mc->id_priv;
> +		evt->event.id = &evt->id_priv->id;
> +		ucma_copy_ud_event(evt, &resp->param.ud);
> +		evt->event.param.ud.private_data = evt->mc->context;
> +		evt->event.status = ucma_process_join(evt);
> +		if (evt->event.status)
> +			evt->event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
> +		break;
>  	case RDMA_CM_EVENT_MULTICAST_ERROR:
> -		ucma_process_mcast(&id_priv->id, evt);
> +		evt->mc = (void *) (uintptr_t) resp->uid;
> +		evt->id_priv = evt->mc->id_priv;
> +		evt->event.id = &evt->id_priv->id;
> +		evt->event.status = resp->status;
> +		evt->event.param.ud.private_data = evt->mc->context;
>  		break;
>  	default:
> +		evt->id_priv = (void *) (uintptr_t) resp->uid;
> +		evt->event.id = &evt->id_priv->id;
> +		if (evt->id_priv->id.ps == RDMA_PS_TCP)
> +			ucma_copy_conn_event(evt, &resp->param.conn);
> +		else
> +			ucma_copy_ud_event(evt, &resp->param.ud);
>  		break;
>  	}
>  
> -	*event = evt;
> -	return 0;
> -}
> -
> -int rdma_get_option(struct rdma_cm_id *id, int level, int optname,
> -		    void *optval, size_t *optlen)
> -{
> -	struct ucma_abi_get_option_resp *resp;
> -	struct ucma_abi_get_option *cmd;
> -	struct cma_id_private *id_priv;
> -	void *msg;
> -	int ret, size;
> -	
> -	CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_GET_OPTION, size);
> -	id_priv = container_of(id, struct cma_id_private, id);
> -	cmd->id = id_priv->handle;
> -	cmd->optval = (uintptr_t) optval;
> -	cmd->level = level;
> -	cmd->optname = optname;
> -	cmd->optlen = *optlen;
> -
> -	ret = write(id->channel->fd, msg, size);
> -	if (ret != size)
> -		return (ret > 0) ? -ENODATA : ret;
> -
> -	*optlen = resp->optlen;
> -	return 0;
> -}
> -
> -int rdma_set_option(struct rdma_cm_id *id, int level, int optname,
> -		    void *optval, size_t optlen)
> -{
> -	struct ucma_abi_set_option *cmd;
> -	struct cma_id_private *id_priv;
> -	void *msg;
> -	int ret, size;
> -	
> -	CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_SET_OPTION, size);
> -	id_priv = container_of(id, struct cma_id_private, id);
> -	cmd->id = id_priv->handle;
> -	cmd->optval = (uintptr_t) optval;
> -	cmd->level = level;
> -	cmd->optname = optname;
> -	cmd->optlen = optlen;
> -
> -	ret = write(id->channel->fd, msg, size);
> -	if (ret != size)
> -		return (ret > 0) ? -ENODATA : ret;
> -
> -	return 0;
> -}
> -
> -int rdma_get_dst_attr(struct rdma_cm_id *id, struct sockaddr *addr,
> -		      struct ibv_ah_attr *ah_attr, uint32_t *remote_qpn,
> -		      uint32_t *remote_qkey)
> -{
> -	struct ucma_abi_dst_attr_resp *resp;
> -	struct ucma_abi_get_dst_attr *cmd;
> -	struct cma_id_private *id_priv;
> -	void *msg;
> -	int ret, size, addrlen;
> -	
> -	addrlen = ucma_addrlen(addr);
> -	if (!addrlen)
> -		return -EINVAL;
> -
> -	CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_GET_DST_ATTR, size);
> -	id_priv = container_of(id, struct cma_id_private, id);
> -	cmd->id = id_priv->handle;
> -	memcpy(&cmd->addr, addr, addrlen);
> -
> -	ret = write(id->channel->fd, msg, size);
> -	if (ret != size)
> -		return (ret > 0) ? -ENODATA : ret;
> -
> -	ibv_copy_ah_attr_from_kern(ah_attr, &resp->ah_attr);
> -	*remote_qpn = resp->remote_qpn;
> -	*remote_qkey = resp->remote_qkey;
> +	*event = &evt->event;
>  	return 0;
>  }
> Index: Makefile.am
> ===================================================================
> --- Makefile.am	(revision 9192)
> +++ Makefile.am	(working copy)
> @@ -31,12 +31,10 @@ examples_mckey_LDADD = $(top_builddir)/s
>  librdmacmincludedir = $(includedir)/rdma
>  
>  librdmacminclude_HEADERS = include/rdma/rdma_cma_abi.h \
> -			   include/rdma/rdma_cma.h \
> -			   include/rdma/rdma_cma_ib.h
> +			   include/rdma/rdma_cma.h
>  
>  EXTRA_DIST = include/rdma/rdma_cma_abi.h \
>  	     include/rdma/rdma_cma.h \
> -	     include/rdma/rdma_cma_ib.h \
>  	     src/librdmacm.map \
>  	     librdmacm.spec.in
>  
> Index: examples/mckey.c
> ===================================================================
> --- examples/mckey.c	(revision 9208)
> +++ examples/mckey.c	(working copy)
> @@ -1,5 +1,5 @@
>  /*
> - * Copyright (c) 2005 Intel Corporation.  All rights reserved.
> + * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
>   *
>   * This software is available to you under a choice of one of two
>   * licenses.  You may choose to be licensed under the terms of the GNU
> @@ -42,9 +42,9 @@
>  #include <netdb.h>
>  #include <byteswap.h>
>  #include <unistd.h>
> +#include <getopt.h>
>  
>  #include <rdma/rdma_cma.h>
> -#include <rdma/rdma_cma_ib.h>
>  
>  struct cmatest_node {
>  	int			id;
> @@ -76,6 +76,8 @@ static int connections = 1;
>  static int message_size = 100;
>  static int message_count = 10;
>  static int is_sender;
> +static char *dst_addr;
> +static char *src_addr;
>  
>  static int create_message(struct cmatest_node *node)
>  {
> @@ -239,19 +241,12 @@ err:
>  	return ret;
>  }
>  
> -static int join_handler(struct cmatest_node *node)
> +static int join_handler(struct cmatest_node *node,
> +			struct rdma_ud_param *param)
>  {
> -	struct ibv_ah_attr ah_attr;
> -	int ret;
> -
> -	ret = rdma_get_dst_attr(node->cma_id, test.dst_addr, &ah_attr,
> -				&node->remote_qpn, &node->remote_qkey);
> -	if (ret) {
> -		printf("mckey: failure getting destination attributes\n");
> -		goto err;
> -	}
> -
> -	node->ah = ibv_create_ah(node->pd, &ah_attr);
> +	node->remote_qpn = param->qp_num;
> +	node->remote_qkey = param->qkey;
> +	node->ah = ibv_create_ah(node->pd, &param->ah_attr);
>  	if (!node->ah) {
>  		printf("mckey: failure creating address handle\n");
>  		goto err;
> @@ -262,7 +257,7 @@ static int join_handler(struct cmatest_n
>  	return 0;
>  err:
>  	connect_error();
> -	return ret;
> +	return -1;
>  }
>  
>  static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
> @@ -274,7 +269,7 @@ static int cma_handler(struct rdma_cm_id
>  		ret = addr_handler(cma_id->context);
>  		break;
>  	case RDMA_CM_EVENT_MULTICAST_JOIN:
> -		ret = join_handler(cma_id->context);
> +		ret = join_handler(cma_id->context, &event->param.ud);
>  		break;
>  	case RDMA_CM_EVENT_ADDR_ERROR:
>  	case RDMA_CM_EVENT_ROUTE_ERROR:
> @@ -411,18 +406,21 @@ out:
>  	return ret;
>  }
>  
> -static int run(char *dst, char *src)
> +static int run(void)
>  {
>  	int i, ret;
>  
> -	printf("mckey: starting client\n");
> -	if (src) {
> -		ret = get_addr(src, &test.src_in);
> +	if (is_sender)
> +		printf("mckey: starting client\n");
> +	else
> +		printf("mckey: starting server\n");
> +	if (src_addr) {
> +		ret = get_addr(src_addr, &test.src_in);
>  		if (ret)
>  			return ret;
>  	}
>  
> -	ret = get_addr(dst, &test.dst_in);
> +	ret = get_addr(dst_addr, &test.dst_in);
>  	if (ret)
>  		return ret;
>  
> @@ -431,7 +429,7 @@ static int run(char *dst, char *src)
>  	printf("mckey: joining\n");
>  	for (i = 0; i < connections; i++) {
>  		ret = rdma_resolve_addr(test.nodes[i].cma_id,
> -					src ? test.src_addr : NULL,
> +					src_addr ? test.src_addr : NULL,
>  					test.dst_addr, 2000);
>  		if (ret) {
>  			printf("mckey: failure getting addr: %d\n", ret);
> @@ -472,14 +470,39 @@ out:
>  
>  int main(int argc, char **argv)
>  {
> -	int ret;
> +	int op, ret;
>  
> -	if (argc < 3 || argc > 4) {
> -		printf("usage: %s {s[end] | r[ecv]} mcast_addr [bind_addr]]\n",
> -		       argv[0]);
> -		exit(1);
> +	while ((op = getopt(argc, argv, "m:sb:c:C:S:")) != -1) {
> +		switch (op) {
> +		case 'm':
> +			dst_addr = optarg;
> +			break;
> +		case 's':
> +			is_sender = 1;
> +			break;
> +		case 'b':
> +			src_addr = optarg;
> +			break;
> +		case 'c':
> +			connections = atoi(optarg);
> +			break;
> +		case 'C':
> +			message_count = atoi(optarg);
> +			break;
> +		case 'S':
> +			message_size = atoi(optarg);
> +			break;
> +		default:
> +			printf("usage: %s\n", argv[0]);
> +			printf("\t-m multicast_address\n");
> +			printf("\t[-s(ender)]\n");
> +			printf("\t[-b bind_address]\n");
> +			printf("\t[-c connections]\n");
> +			printf("\t[-C message_count]\n");
> +			printf("\t[-S message_size]\n");
> +			exit(1);
> +		}
>  	}
> -	is_sender = (argv[1][0] == 's');
>  
>  	test.dst_addr = (struct sockaddr *) &test.dst_in;
>  	test.src_addr = (struct sockaddr *) &test.src_in;
> @@ -494,7 +517,7 @@ int main(int argc, char **argv)
>  	if (alloc_nodes())
>  		exit(1);
>  
> -	ret = run(argv[2], (argc == 4) ? argv[3] : NULL);
> +	ret = run();
>  
>  	printf("test complete\n");
>  	destroy_nodes();
> Index: examples/udaddy.c
> ===================================================================
> --- examples/udaddy.c	(revision 9208)
> +++ examples/udaddy.c	(working copy)
> @@ -1,5 +1,5 @@
>  /*
> - * Copyright (c) 2005 Intel Corporation.  All rights reserved.
> + * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
>   *
>   * This software is available to you under a choice of one of two
>   * licenses.  You may choose to be licensed under the terms of the GNU
> @@ -41,15 +41,9 @@
>  #include <sys/socket.h>
>  #include <netdb.h>
>  #include <byteswap.h>
> +#include <getopt.h>
>  
>  #include <rdma/rdma_cma.h>
> -#include <rdma/rdma_cma_ib.h>
> -
> -/*
> - * To execute:
> - * Server: udaddy
> - * Client: udaddy [server_addr [src_addr]]
> - */
>  
>  struct cmatest_node {
>  	int			id;
> @@ -80,7 +74,8 @@ static struct cmatest test;
>  static int connections = 1;
>  static int message_size = 100;
>  static int message_count = 10;
> -static int is_server;
> +static char *dst_addr;
> +static char *src_addr;
>  
>  static int create_message(struct cmatest_node *node)
>  {
> @@ -246,7 +241,6 @@ static int route_handler(struct cmatest_
>  
>  	memset(&conn_param, 0, sizeof conn_param);
>  	conn_param.qp_num = node->cma_id->qp->qp_num;
> -	conn_param.qp_type = node->cma_id->qp->qp_type;
>  	conn_param.retry_count = 5;
>  	ret = rdma_connect(node->cma_id, &conn_param);
>  	if (ret) {
> @@ -284,7 +278,6 @@ static int connect_handler(struct rdma_c
>  
>  	memset(&conn_param, 0, sizeof conn_param);
>  	conn_param.qp_num = node->cma_id->qp->qp_num;
> -	conn_param.qp_type = node->cma_id->qp->qp_type;
>  	ret = rdma_accept(node->cma_id, &conn_param);
>  	if (ret) {
>  		printf("udaddy: failure accepting: %d\n", ret);
> @@ -303,19 +296,12 @@ err1:
>  	return ret;
>  }
>  
> -static int resolved_handler(struct cmatest_node *node)
> +static int resolved_handler(struct cmatest_node *node,
> +			    struct rdma_cm_event *event)
>  {
> -	struct ibv_ah_attr ah_attr;
> -	int ret;
> -
> -	ret = rdma_get_dst_attr(node->cma_id, test.dst_addr, &ah_attr,
> -				&node->remote_qpn, &node->remote_qkey);
> -	if (ret) {
> -		printf("udaddy: failure getting destination attributes\n");
> -		goto err;
> -	}
> -
> -	node->ah = ibv_create_ah(node->pd, &ah_attr);
> +	node->remote_qpn = event->param.ud.qp_num;
> +	node->remote_qkey = event->param.ud.qkey;
> +	node->ah = ibv_create_ah(node->pd, &event->param.ud.ah_attr);
>  	if (!node->ah) {
>  		printf("udaddy: failure creating address handle\n");
>  		goto err;
> @@ -326,7 +312,7 @@ static int resolved_handler(struct cmate
>  	return 0;
>  err:
>  	connect_error();
> -	return ret;
> +	return -1;
>  }
>  
>  static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
> @@ -344,7 +330,7 @@ static int cma_handler(struct rdma_cm_id
>  		ret = connect_handler(cma_id);
>  		break;
>  	case RDMA_CM_EVENT_ESTABLISHED:
> -		ret = resolved_handler(cma_id->context);
> +		ret = resolved_handler(cma_id->context, event);
>  		break;
>  	case RDMA_CM_EVENT_ADDR_ERROR:
>  	case RDMA_CM_EVENT_ROUTE_ERROR:
> @@ -404,7 +390,7 @@ static int alloc_nodes(void)
>  
>  	for (i = 0; i < connections; i++) {
>  		test.nodes[i].id = i;
> -		if (!is_server) {
> +		if (dst_addr) {
>  			ret = rdma_create_id(test.channel,
>  					     &test.nodes[i].cma_id,
>  					     &test.nodes[i], RDMA_PS_UDP);
> @@ -475,6 +461,28 @@ static int connect_events(void)
>  	return ret;
>  }
>  
> +static int get_addr(char *dst, struct sockaddr_in *addr)
> +{
> +	struct addrinfo *res;
> +	int ret;
> +
> +	ret = getaddrinfo(dst, NULL, NULL, &res);
> +	if (ret) {
> +		printf("getaddrinfo failed - invalid hostname or IP address\n");
> +		return ret;
> +	}
> +
> +	if (res->ai_family != PF_INET) {
> +		ret = -1;
> +		goto out;
> +	}
> +
> +	*addr = *(struct sockaddr_in *) res->ai_addr;
> +out:
> +	freeaddrinfo(res);
> +	return ret;
> +}
> +
>  static int run_server(void)
>  {
>  	struct rdma_cm_id *listen_id;
> @@ -487,7 +495,13 @@ static int run_server(void)
>  		return ret;
>  	}
>  
> -	test.src_in.sin_family = PF_INET;
> +	if (src_addr) {
> +		ret = get_addr(src_addr, &test.src_in);
> +		if (ret)
> +			goto out;
> +	} else
> +		test.src_in.sin_family = PF_INET;
> +
>  	test.src_in.sin_port = 7174;
>  	ret = rdma_bind_addr(listen_id, test.src_addr);
>  	if (ret) {
> @@ -526,40 +540,18 @@ out:
>  	return ret;
>  }
>  
> -static int get_addr(char *dst, struct sockaddr_in *addr)
> -{
> -	struct addrinfo *res;
> -	int ret;
> -
> -	ret = getaddrinfo(dst, NULL, NULL, &res);
> -	if (ret) {
> -		printf("getaddrinfo failed - invalid hostname or IP address\n");
> -		return ret;
> -	}
> -
> -	if (res->ai_family != PF_INET) {
> -		ret = -1;
> -		goto out;
> -	}
> -
> -	*addr = *(struct sockaddr_in *) res->ai_addr;
> -out:
> -	freeaddrinfo(res);
> -	return ret;
> -}
> -
> -static int run_client(char *dst, char *src)
> +static int run_client(void)
>  {
>  	int i, ret;
>  
>  	printf("udaddy: starting client\n");
> -	if (src) {
> -		ret = get_addr(src, &test.src_in);
> +	if (src_addr) {
> +		ret = get_addr(src_addr, &test.src_in);
>  		if (ret)
>  			return ret;
>  	}
>  
> -	ret = get_addr(dst, &test.dst_in);
> +	ret = get_addr(dst_addr, &test.dst_in);
>  	if (ret)
>  		return ret;
>  
> @@ -568,7 +560,7 @@ static int run_client(char *dst, char *s
>  	printf("udaddy: connecting\n");
>  	for (i = 0; i < connections; i++) {
>  		ret = rdma_resolve_addr(test.nodes[i].cma_id,
> -					src ? test.src_addr : NULL,
> +					src_addr ? test.src_addr : NULL,
>  					test.dst_addr, 2000);
>  		if (ret) {
>  			printf("udaddy: failure getting addr: %d\n", ret);
> @@ -601,13 +593,35 @@ out:
>  
>  int main(int argc, char **argv)
>  {
> -	int ret;
> +	int op, ret;
>  
> -	if (argc > 3) {
> -		printf("usage: %s [server_addr [src_addr]]\n", argv[0]);
> -		exit(1);
> +	while ((op = getopt(argc, argv, "s:b:c:C:S:")) != -1) {
> +		switch (op) {
> +		case 's':
> +			dst_addr = optarg;
> +			break;
> +		case 'b':
> +			src_addr = optarg;
> +			break;
> +		case 'c':
> +			connections = atoi(optarg);
> +			break;
> +		case 'C':
> +			message_count = atoi(optarg);
> +			break;
> +		case 'S':
> +			message_size = atoi(optarg);
> +			break;
> +		default:
> +			printf("usage: %s\n", argv[0]);
> +			printf("\t[-s server_address]\n");
> +			printf("\t[-b bind_address]\n");
> +			printf("\t[-c connections]\n");
> +			printf("\t[-C message_count]\n");
> +			printf("\t[-S message_size]\n");
> +			exit(1);
> +		}
>  	}
> -	is_server = (argc == 1);
>  
>  	test.dst_addr = (struct sockaddr *) &test.dst_in;
>  	test.src_addr = (struct sockaddr *) &test.src_in;
> @@ -622,10 +636,10 @@ int main(int argc, char **argv)
>  	if (alloc_nodes())
>  		exit(1);
>  
> -	if (is_server)
> -		ret = run_server();
> +	if (dst_addr)
> +		ret = run_client();
>  	else
> -		ret = run_client(argv[1], (argc == 3) ? argv[2] : NULL);
> +		ret = run_server();
>  
>  	printf("test complete\n");
>  	destroy_nodes();
> Index: examples/cmatose.c
> ===================================================================
> --- examples/cmatose.c	(revision 9192)
> +++ examples/cmatose.c	(working copy)
> @@ -1,5 +1,5 @@
>  /*
> - * Copyright (c) 2005 Intel Corporation.  All rights reserved.
> + * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
>   *
>   * This software is available to you under a choice of one of two
>   * licenses.  You may choose to be licensed under the terms of the GNU
> @@ -41,6 +41,7 @@
>  #include <sys/socket.h>
>  #include <netdb.h>
>  #include <byteswap.h>
> +#include <getopt.h>
>  
>  #include <rdma/rdma_cma.h>
>  
> @@ -52,12 +53,6 @@ static inline uint64_t cpu_to_be64(uint6
>  static inline uint32_t cpu_to_be32(uint32_t x) { return bswap_32(x); }
>  #endif
>  
> -/*
> - * To execute:
> - * Server: rdma_cmatose
> - * Client: rdma_cmatose <dst_ip>
> - */
> -
>  struct cmatest_node {
>  	int			id;
>  	struct rdma_cm_id	*cma_id;
> @@ -85,7 +80,8 @@ static struct cmatest test;
>  static int connections = 1;
>  static int message_size = 100;
>  static int message_count = 10;
> -static int is_server;
> +static char *dst_addr;
> +static char *src_addr;
>  
>  static int create_message(struct cmatest_node *node)
>  {
> @@ -377,7 +373,7 @@ static int alloc_nodes(void)
>  
>  	for (i = 0; i < connections; i++) {
>  		test.nodes[i].id = i;
> -		if (!is_server) {
> +		if (dst_addr) {
>  			ret = rdma_create_id(test.channel,
>  					     &test.nodes[i].cma_id,
>  					     &test.nodes[i], RDMA_PS_TCP);
> @@ -460,6 +456,28 @@ static int disconnect_events(void)
>  	return ret;
>  }
>  
> +static int get_addr(char *dst, struct sockaddr_in *addr)
> +{
> +	struct addrinfo *res;
> +	int ret;
> +
> +	ret = getaddrinfo(dst, NULL, NULL, &res);
> +	if (ret) {
> +		printf("getaddrinfo failed - invalid hostname or IP address\n");
> +		return ret;
> +	}
> +
> +	if (res->ai_family != PF_INET) {
> +		ret = -1;
> +		goto out;
> +	}
> +
> +	*addr = *(struct sockaddr_in *) res->ai_addr;
> +out:
> +	freeaddrinfo(res);
> +	return ret;
> +}
> +
>  static int run_server(void)
>  {
>  	struct rdma_cm_id *listen_id;
> @@ -472,12 +490,18 @@ static int run_server(void)
>  		return ret;
>  	}
>  
> -	test.src_in.sin_family = PF_INET;
> +	if (src_addr) {
> +		ret = get_addr(src_addr, &test.src_in);
> +		if (ret)
> +			goto out;
> +	} else
> +		test.src_in.sin_family = PF_INET;
> +
>  	test.src_in.sin_port = 7471;
>  	ret = rdma_bind_addr(listen_id, test.src_addr);
>  	if (ret) {
>  		printf("cmatose: bind address failed: %d\n", ret);
> -		return ret;
> +		goto out;
>  	}
>  
>  	ret = rdma_listen(listen_id, 0);
> @@ -528,40 +552,18 @@ out:
>  	return ret;
>  }
>  
> -static int get_addr(char *dst, struct sockaddr_in *addr)
> -{
> -	struct addrinfo *res;
> -	int ret;
> -
> -	ret = getaddrinfo(dst, NULL, NULL, &res);
> -	if (ret) {
> -		printf("getaddrinfo failed - invalid hostname or IP address\n");
> -		return ret;
> -	}
> -
> -	if (res->ai_family != PF_INET) {
> -		ret = -1;
> -		goto out;
> -	}
> -
> -	*addr = *(struct sockaddr_in *) res->ai_addr;
> -out:
> -	freeaddrinfo(res);
> -	return ret;
> -}
> -
> -static int run_client(char *dst, char *src)
> +static int run_client(void)
>  {
>  	int i, ret, ret2;
>  
>  	printf("cmatose: starting client\n");
> -	if (src) {
> -		ret = get_addr(src, &test.src_in);
> +	if (src_addr) {
> +		ret = get_addr(src_addr, &test.src_in);
>  		if (ret)
>  			return ret;
>  	}
>  
> -	ret = get_addr(dst, &test.dst_in);
> +	ret = get_addr(dst_addr, &test.dst_in);
>  	if (ret)
>  		return ret;
>  
> @@ -570,7 +572,7 @@ static int run_client(char *dst, char *s
>  	printf("cmatose: connecting\n");
>  	for (i = 0; i < connections; i++) {
>  		ret = rdma_resolve_addr(test.nodes[i].cma_id,
> -					src ? test.src_addr : NULL,
> +					src_addr ? test.src_addr : NULL,
>  					test.dst_addr, 2000);
>  		if (ret) {
>  			printf("cmatose: failure getting addr: %d\n", ret);
> @@ -597,7 +599,6 @@ static int run_client(char *dst, char *s
>  		}
>  
>  		printf("data transfers complete\n");
> -
>  	}
>  
>  	ret = 0;
> @@ -611,13 +612,35 @@ out:
>  
>  int main(int argc, char **argv)
>  {
> -	int ret;
> +	int op, ret;
>  
> -	if (argc > 3) {
> -		printf("usage: %s [server_addr [src_addr]]\n", argv[0]);
> -		exit(1);
> +	while ((op = getopt(argc, argv, "s:b:c:C:S:")) != -1) {
> +		switch (op) {
> +		case 's':
> +			dst_addr = optarg;
> +			break;
> +		case 'b':
> +			src_addr = optarg;
> +			break;
> +		case 'c':
> +			connections = atoi(optarg);
> +			break;
> +		case 'C':
> +			message_count = atoi(optarg);
> +			break;
> +		case 'S':
> +			message_size = atoi(optarg);
> +			break;
> +		default:
> +			printf("usage: %s\n", argv[0]);
> +			printf("\t[-s server_address]\n");
> +			printf("\t[-b bind_address]\n");
> +			printf("\t[-c connections]\n");
> +			printf("\t[-C message_count]\n");
> +			printf("\t[-S message_size]\n");
> +			exit(1);
> +		}
>  	}
> -	is_server = (argc == 1);
>  
>  	test.dst_addr = (struct sockaddr *) &test.dst_in;
>  	test.src_addr = (struct sockaddr *) &test.src_in;
> @@ -633,10 +656,10 @@ int main(int argc, char **argv)
>  	if (alloc_nodes())
>  		exit(1);
>  
> -	if (is_server)
> -		ret = run_server();
> +	if (dst_addr)
> +		ret = run_client();
>  	else
> -		ret = run_client(argv[1], (argc == 3) ? argv[2] : NULL);
> +		ret = run_server();
>  
>  	printf("test complete\n");
>  	destroy_nodes();
> 
> 
> _______________________________________________
> openib-general mailing list
> openib-general at openib.org
> http://openib.org/mailman/listinfo/openib-general
> 
> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
> 





More information about the general mailing list