[ofa-general] Re: [PATCH] ibsim: Add better end port simulation support

Sasha Khapyorsky sashak at voltaire.com
Tue Feb 17 13:18:48 PST 2009


Hi Hal,

On 15:37 Sat 14 Feb     , hnrose at comcast.net wrote:
> 
> Add SIM_PORT environment variable to allow for end port selection

How this would handle case when SIM_PORT=N, but program tries to work
via another port (for example: SIM_PORT=2 and ibnetdiscover -P 1)?

IOW should port number selection be initiated natively by program rather
than by using environment variables?

> Signed-off-by: Hal Rosenstock <hal.rosenstock at gmail.com>
> ---
>  ibsim/ibsim.c         |    6 +-
>  include/ibsim.h       |    2 +
>  umad2sim/sim_client.c |   49 +++++++++-
>  umad2sim/sim_client.h |    4 +-
>  umad2sim/umad2sim.c   |  254 ++++++++++++++++++++++++++-----------------------
>  5 files changed, 189 insertions(+), 126 deletions(-)
> 
> diff --git a/ibsim/ibsim.c b/ibsim/ibsim.c
> index f48e1f0..6a35fdc 100644
> --- a/ibsim/ibsim.c
> +++ b/ibsim/ibsim.c
> @@ -1,5 +1,6 @@
>  /*
>   * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
> + * Copyright (c) 2009 HNR Consulting. All rights reserved.
>   *
>   * This file is part of ibsim.
>   *
> @@ -187,7 +188,8 @@ static int sm_exists(Node * node)
>  	return 0;
>  }
>  
> -static int sim_ctl_new_client(Client * cl, struct sim_ctl * ctl, union name_t *from)
> +static int sim_ctl_new_client(Client * cl, struct sim_ctl * ctl,
> +			      union name_t *from)
>  {
>  	union name_t name;
>  	size_t size;
> @@ -219,7 +221,7 @@ static int sim_ctl_new_client(Client * cl, struct sim_ctl * ctl, union name_t *f
>  			ctl->type = SIM_CTL_ERROR;
>  			return -1;
>  		}
> -		cl->port = node_get_port(node, 0);
> +		cl->port = node_get_port(node, scl->portnum);
>  		VERB("Attaching client %d at node \"%s\" port 0x%" PRIx64,
>  		     i, node->nodeid, cl->port->portguid);
>  	} else {
> diff --git a/include/ibsim.h b/include/ibsim.h
> index 15fc37c..66ba6f9 100644
> --- a/include/ibsim.h
> +++ b/include/ibsim.h
> @@ -1,5 +1,6 @@
>  /*
>   * Copyright (c) 2006-2008 Voltaire, Inc. All rights reserved.
> + * Copyright (c) 2009 HNR Consulting. All rights reserved.
>   *
>   * This file is part of ibsim.
>   *
> @@ -100,6 +101,7 @@ struct sim_client_info {
>  	uint32_t qp;
>  	uint32_t issm;		/* accept request for qp 0 & 1 */
>  	char nodeid[32];
> +	uint32_t portnum;
>  };
>  
>  union name_t {
> diff --git a/umad2sim/sim_client.c b/umad2sim/sim_client.c
> index 06bb7a8..1c35109 100644
> --- a/umad2sim/sim_client.c
> +++ b/umad2sim/sim_client.c
> @@ -1,5 +1,6 @@
>  /*
>   * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
> + * Copyright (c) 2009 HNR Consulting. All rights reserved.
>   *
>   * This file is part of ibsim.
>   *
> @@ -182,6 +183,7 @@ static int sim_connect(struct sim_client *sc, int id, int qp, char *nodeid)
>  	info.id = id;
>  	info.issm = 0;
>  	info.qp = qp;
> +	info.portnum = sc->portnum;
>  
>  	if (nodeid)
>  		strncpy(info.nodeid, nodeid, sizeof(info.nodeid) - 1);
> @@ -202,7 +204,7 @@ static int sim_disconnect(struct sim_client *sc)
>  	return sim_ctl(sc, SIM_CTL_DISCONNECT, 0, 0);
>  }
>  
> -static int sim_init(struct sim_client *sc, char *nodeid)
> +static int sim_init(struct sim_client *sc, char *nodeid, int portnum)
>  {
>  	union name_t name;
>  	socklen_t size;
> @@ -238,6 +240,7 @@ static int sim_init(struct sim_client *sc, char *nodeid)
>  	DEBUG("init %d: opened ctl fd %d as \'%s\'",
>  	      pid, ctlfd, get_name(&name));
>  
> +	sc->portnum = portnum;
>  	port = connect_port ? atoi(connect_port) : IBSIM_DEFAULT_SERVER_PORT;
>  	size = make_name(&name, connect_host, port, "%s:ctl", socket_basename);
>  
> @@ -286,9 +289,17 @@ int sim_client_set_sm(struct sim_client *sc, unsigned issm)
>  int sim_client_init(struct sim_client *sc)
>  {
>  	char *nodeid;
> +	char *portno;
> +	int i, j = 0, portnum = 0, startport = 1, endport;
> +	uint8_t numports, nodetype;
> +	uint8_t *portinfo;
>  
>  	nodeid = getenv("SIM_HOST");
> -	if (sim_init(sc, nodeid) < 0)
> +	portno = getenv("SIM_PORT");
> +	if (portno)
> +		portnum = atoi(portno);
> +
> +	if (sim_init(sc, nodeid, portnum) < 0)
>  		return -1;
>  	if (sim_ctl(sc, SIM_CTL_GET_VENDOR, &sc->vendor,
>  		    sizeof(sc->vendor)) < 0)
> @@ -296,11 +307,37 @@ int sim_client_init(struct sim_client *sc)
>  	if (sim_ctl(sc, SIM_CTL_GET_NODEINFO, sc->nodeinfo,
>  		    sizeof(sc->nodeinfo)) < 0)
>  		goto _exit;
> +	numports = mad_get_field(sc->nodeinfo, 0, IB_NODE_NPORTS_F);
> +	nodetype = mad_get_field(sc->nodeinfo, 0, IB_NODE_TYPE_F);
> +	if (nodetype == 2) { // switch
> +		startport = 0;
> +		endport = 0;
> +	} else {
> +		if (portnum == 0) {
> +			IBWARN("portnum 0 is not valid end port on non switch node");
> +			goto _exit;
> +		}

This makes exporting SIM_PORT environment variable to be mandatory,
which doesn't look like a good idea for me (personally I will need to
rewrite some amount of my scripts).

I think that SIM_HOST should be optional and the default behavior
should be preserved.

> +		endport = numports;
> +	}
> +	if (portnum > endport) {
> +		IBWARN("portnum %d is not a valid end port number (%d)",
> +		       portnum, endport);
> +		goto _exit;
> +	}
>  
> -	sc->portinfo[0] = 0;	// portno requested
> -	if (sim_ctl(sc, SIM_CTL_GET_PORTINFO, sc->portinfo,
> -		    sizeof(sc->portinfo)) < 0)
> +	sc->portinfo = malloc(64 * (nodetype != 2 ? numports + 1 : 1));	// portinfo size x number of ports starting at 0
> +	if (!sc->portinfo)
>  		goto _exit;
> +
> +	// loop through end ports
> +	for (i = startport; i <= endport ; i++, j++) {
> +		portinfo = sc->portinfo + 64 * j;

You don't need 'j' - just move portinfo pointer.

> +		*portinfo = i + 1; // portno requested
> +		if (sim_ctl(sc, SIM_CTL_GET_PORTINFO, portinfo, 64) < 0)
> +			goto _exit;
> +	}
> +
> +	// although pkeys also per port, current config same on all end ports

Which is not correct really.

Sasha

>  	if (sim_ctl(sc, SIM_CTL_GET_PKEYS, sc->pkeys, sizeof(sc->pkeys)) < 0)
>  		goto _exit;
>  	if (getenv("SIM_SET_ISSM"))
> @@ -315,5 +352,7 @@ int sim_client_init(struct sim_client *sc)
>  void sim_client_exit(struct sim_client *sc)
>  {
>  	sim_disconnect(sc);
> +	if (sc->portinfo)
> +		free(sc->portinfo);
>  	sc->fd_ctl = sc->fd_pktin = sc->fd_pktout = -1;
>  }
> diff --git a/umad2sim/sim_client.h b/umad2sim/sim_client.h
> index 80ed442..0faca80 100644
> --- a/umad2sim/sim_client.h
> +++ b/umad2sim/sim_client.h
> @@ -1,5 +1,6 @@
>  /*
>   * Copyright (c) 2006,2007 Voltaire, Inc. All rights reserved.
> + * Copyright (c) 2009 HNR Consulting. All rights reserved.
>   *
>   * This file is part of ibsim.
>   *
> @@ -41,8 +42,9 @@ struct sim_client {
>  	int clientid;
>  	int fd_pktin, fd_pktout, fd_ctl;
>  	struct sim_vendor vendor;
> +	int portnum;
>  	uint8_t nodeinfo[64];
> -	uint8_t portinfo[64];
> +	uint8_t *portinfo;
>  	uint16_t pkeys[SIM_CTL_MAX_DATA/sizeof(uint16_t)];
>  };
>  
> diff --git a/umad2sim/umad2sim.c b/umad2sim/umad2sim.c
> index 8d83a24..6e3c269 100644
> --- a/umad2sim/umad2sim.c
> +++ b/umad2sim/umad2sim.c
> @@ -1,5 +1,6 @@
>  /*
>   * Copyright (c) 2006-2008 Voltaire, Inc. All rights reserved.
> + * Copyright (c) 2009 HNR Consulting. All rights reserved.
>   *
>   * This file is part of ibsim.
>   *
> @@ -179,7 +180,10 @@ static int dev_sysfs_create(struct umad2sim_dev *dev)
>  	struct sim_client *sc = &dev->sim_client;
>  	char *str;
>  	uint8_t *portinfo;
> -	int i;
> +	char *ports_path_end;
> +	int i, j;
> +	int startport = 1, endport;
> +	uint8_t numports, nodetype;
>  
>  	/* /sys/class/infiniband_mad/abi_version */
>  	snprintf(path, sizeof(path), "%s", sysfs_infiniband_mad_dir);
> @@ -232,123 +236,138 @@ static int dev_sysfs_create(struct umad2sim_dev *dev)
>  	strncat(path, "/ports", sizeof(path) - 1);
>  	make_path(path);
>  
> -	portinfo = sc->portinfo;
> -
> -	/* /sys/class/infiniband/mthca0/ports/1/ */
> -	val = mad_get_field(portinfo, 0, IB_PORT_LOCAL_PORT_F);
> -	snprintf(path + strlen(path), sizeof(path) - strlen(path), "/%u", val);
> -	make_path(path);
> -
> -	/* /sys/class/infiniband/mthca0/ports/1/lid_mask_count */
> -	val = mad_get_field(portinfo, 0, IB_PORT_LMC_F);
> -	file_printf(path, SYS_PORT_LMC, "%d", val);
> -
> -	/* /sys/class/infiniband/mthca0/ports/1/sm_lid */
> -	val = mad_get_field(portinfo, 0, IB_PORT_SMLID_F);
> -	file_printf(path, SYS_PORT_SMLID, "0x%x", val);
> -
> -	/* /sys/class/infiniband/mthca0/ports/1/sm_sl */
> -	val = mad_get_field(portinfo, 0, IB_PORT_SMSL_F);
> -	file_printf(path, SYS_PORT_SMSL, "%d", val);
> -
> -	/* /sys/class/infiniband/mthca0/ports/1/lid */
> -	val = mad_get_field(portinfo, 0, IB_PORT_LID_F);
> -	file_printf(path, SYS_PORT_LID, "0x%x", val);
> -
> -	/* /sys/class/infiniband/mthca0/ports/1/state */
> -	val = mad_get_field(portinfo, 0, IB_PORT_STATE_F);
> -	if (val == 0)
> -		str = "NOP";
> -	else if (val == 1)
> -		str = "DOWN";
> -	else if (val == 2)
> -		str = "INIT";
> -	else if (val == 3)
> -		str = "ARMED";
> -	else if (val == 4)
> -		str = "ACTIVE";
> -	else if (val == 5)
> -		str = "ACTIVE_DEFER";
> -	else
> -		str = "<unknown>";
> -	file_printf(path, SYS_PORT_STATE, "%d: %s\n", val, str);
> -
> -	/* /sys/class/infiniband/mthca0/ports/1/phys_state */
> -	val = mad_get_field(portinfo, 0, IB_PORT_PHYS_STATE_F);
> -	if (val == 1)
> -		str = "Sleep";
> -	else if (val == 2)
> -		str = "Polling";
> -	else if (val == 3)
> -		str = "Disabled";
> -	else if (val == 4)
> -		str = "PortConfigurationTraining";
> -	else if (val == 5)
> -		str = "LinkUp";
> -	else if (val == 6)
> -		str = "LinkErrorRecovery";
> -	else if (val == 7)
> -		str = "Phy Test";
> -	else
> -		str = "<unknown>";
> -	file_printf(path, SYS_PORT_PHY_STATE, "%d: %s\n", val, str);
> -
> -	/* /sys/class/infiniband/mthca0/ports/1/rate */
> -	val = mad_get_field(portinfo, 0, IB_PORT_LINK_WIDTH_ACTIVE_F);
> -	speed = mad_get_field(portinfo, 0, IB_PORT_LINK_SPEED_ACTIVE_F);
> -	if (val == 1)
> -		val = 1;
> -	else if (val == 2)
> -		val = 4;
> -	else if (val == 4)
> -		val = 8;
> -	else if (val == 8)
> -		val = 12;
> -	else
> -		val = 0;
> -	if (speed == 2)
> -		str = " DDR";
> -	else if (speed == 4)
> -		str = " QDR";
> -	else
> -		str = "";
> -	file_printf(path, SYS_PORT_RATE, "%d%s Gb/sec (%dX%s)\n",
> -		    (val * speed * 25) / 10,
> -		    (val * speed * 25) % 10 ? ".5" : "", val, str);
> -
> -	/* /sys/class/infiniband/mthca0/ports/1/cap_mask */
> -	val = mad_get_field(portinfo, 0, IB_PORT_CAPMASK_F);
> -	file_printf(path, SYS_PORT_CAPMASK, "0x%08x", val);
> -
> -	/* /sys/class/infiniband/mthca0/ports/1/gids/0 */
> -	str = path + strlen(path);
> -	strncat(path, "/gids", sizeof(path) - 1);
> -	make_path(path);
> -	*str = '\0';
> -	gid = mad_get_field64(portinfo, 0, IB_PORT_GID_PREFIX_F);
> -	guid = mad_get_field64(sc->nodeinfo, 0, IB_NODE_GUID_F) +
> -	    mad_get_field(portinfo, 0, IB_PORT_LOCAL_PORT_F);
> -	file_printf(path, SYS_PORT_GID,
> -		    "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n",
> -		    (uint16_t) ((gid >> 48) & 0xffff),
> -		    (uint16_t) ((gid >> 32) & 0xffff),
> -		    (uint16_t) ((gid >> 16) & 0xffff),
> -		    (uint16_t) ((gid >> 0) & 0xffff),
> -		    (uint16_t) ((guid >> 48) & 0xffff),
> -		    (uint16_t) ((guid >> 32) & 0xffff),
> -		    (uint16_t) ((guid >> 16) & 0xffff),
> -		    (uint16_t) ((guid >> 0) & 0xffff));
> +	numports = mad_get_field(sc->nodeinfo, 0, IB_NODE_NPORTS_F);
> +	nodetype = mad_get_field(sc->nodeinfo, 0, IB_NODE_TYPE_F);
> +        if (nodetype == 2) { // switch
> +		startport = 0;
> +		endport = 0;
> +	} else
> +		endport = numports;
> +
> +	ports_path_end = path + strlen(path);
> +
> +	// loop through end ports
> +	for (j = startport; j <= endport; j++) {
> +
> +		portinfo = sc->portinfo + 64 * j;
> +
> +		/* /sys/class/infiniband/mthca0/ports/<n>/ */
> +		val = mad_get_field(portinfo, 0, IB_PORT_LOCAL_PORT_F);
> +		snprintf(path + strlen(path), sizeof(path) - strlen(path), "/%u", val);
> +		make_path(path);
> +
> +		/* /sys/class/infiniband/mthca0/ports/<n>/lid_mask_count */
> +		val = mad_get_field(portinfo, 0, IB_PORT_LMC_F);
> +		file_printf(path, SYS_PORT_LMC, "%d", val);
> +
> +		/* /sys/class/infiniband/mthca0/ports/<n>/sm_lid */
> +		val = mad_get_field(portinfo, 0, IB_PORT_SMLID_F);
> +		file_printf(path, SYS_PORT_SMLID, "0x%x", val);
> +
> +		/* /sys/class/infiniband/mthca0/ports/<n>/sm_sl */
> +		val = mad_get_field(portinfo, 0, IB_PORT_SMSL_F);
> +		file_printf(path, SYS_PORT_SMSL, "%d", val);
> +
> +		/* /sys/class/infiniband/mthca0/ports/<n>/lid */
> +		val = mad_get_field(portinfo, 0, IB_PORT_LID_F);
> +		file_printf(path, SYS_PORT_LID, "0x%x", val);
> +
> +		/* /sys/class/infiniband/mthca0/ports/<n>/state */
> +		val = mad_get_field(portinfo, 0, IB_PORT_STATE_F);
> +		if (val == 0)
> +			str = "NOP";
> +		else if (val == 1)
> +			str = "DOWN";
> +		else if (val == 2)
> +			str = "INIT";
> +		else if (val == 3)
> +			str = "ARMED";
> +		else if (val == 4)
> +			str = "ACTIVE";
> +		else if (val == 5)
> +			str = "ACTIVE_DEFER";
> +		else
> +			str = "<unknown>";
> +		file_printf(path, SYS_PORT_STATE, "%d: %s\n", val, str);
> +
> +		/* /sys/class/infiniband/mthca0/ports/<n>/phys_state */
> +		val = mad_get_field(portinfo, 0, IB_PORT_PHYS_STATE_F);
> +		if (val == 1)
> +			str = "Sleep";
> +		else if (val == 2)
> +			str = "Polling";
> +		else if (val == 3)
> +			str = "Disabled";
> +		else if (val == 4)
> +			str = "PortConfigurationTraining";
> +		else if (val == 5)
> +			str = "LinkUp";
> +		else if (val == 6)
> +			str = "LinkErrorRecovery";
> +		else if (val == 7)
> +			str = "Phy Test";
> +		else
> +			str = "<unknown>";
> +		file_printf(path, SYS_PORT_PHY_STATE, "%d: %s\n", val, str);
> +
> +		/* /sys/class/infiniband/mthca0/ports/<n>/rate */
> +		val = mad_get_field(portinfo, 0, IB_PORT_LINK_WIDTH_ACTIVE_F);
> +		speed = mad_get_field(portinfo, 0, IB_PORT_LINK_SPEED_ACTIVE_F);
> +		if (val == 1)
> +			val = 1;
> +		else if (val == 2)
> +			val = 4;
> +		else if (val == 4)
> +			val = 8;
> +		else if (val == 8)
> +			val = 12;
> +		else
> +			val = 0;
> +		if (speed == 2)
> +			str = " DDR";
> +		else if (speed == 4)
> +			str = " QDR";
> +		else
> +			str = "";
> +		file_printf(path, SYS_PORT_RATE, "%d%s Gb/sec (%dX%s)\n",
> +			    (val * speed * 25) / 10,
> +			    (val * speed * 25) % 10 ? ".5" : "", val, str);
> +
> +		/* /sys/class/infiniband/mthca0/ports/<n>/cap_mask */
> +		val = mad_get_field(portinfo, 0, IB_PORT_CAPMASK_F);
> +		file_printf(path, SYS_PORT_CAPMASK, "0x%08x", val);
> +
> +		/* /sys/class/infiniband/mthca0/ports/<n>/gids/0 */
> +		str = path + strlen(path);
> +		strncat(path, "/gids", sizeof(path) - 1);
> +		make_path(path);
> +		*str = '\0';
> +		gid = mad_get_field64(portinfo, 0, IB_PORT_GID_PREFIX_F);
> +		guid = mad_get_field64(sc->nodeinfo, 0, IB_NODE_GUID_F) + j;
> +		file_printf(path, SYS_PORT_GID,
> +			    "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n",
> +			    (uint16_t) ((gid >> 48) & 0xffff),
> +			    (uint16_t) ((gid >> 32) & 0xffff),
> +			    (uint16_t) ((gid >> 16) & 0xffff),
> +			    (uint16_t) ((gid >> 0) & 0xffff),
> +			    (uint16_t) ((guid >> 48) & 0xffff),
> +			    (uint16_t) ((guid >> 32) & 0xffff),
> +			    (uint16_t) ((guid >> 16) & 0xffff),
> +			    (uint16_t) ((guid >> 0) & 0xffff));
> +
> +		/* /sys/class/infiniband/mthca0/ports/<n>/pkeys/0 */
> +		str = path + strlen(path);
> +		strncat(path, "/pkeys", sizeof(path) - 1);
> +		make_path(path);
> +		for (i = 0; i < sizeof(sc->pkeys)/sizeof(sc->pkeys[0]); i++) {
> +			char name[8];
> +			snprintf(name, sizeof(name), "%u", i);
> +			file_printf(path, name, "0x%04x\n", ntohs(sc->pkeys[i]));
> +		}
> +		*str = '\0';
>  
> -	/* /sys/class/infiniband/mthca0/ports/1/pkeys/0 */
> -	str = path + strlen(path);
> -	strncat(path, "/pkeys", sizeof(path) - 1);
> -	make_path(path);
> -	for (i = 0; i < sizeof(sc->pkeys)/sizeof(sc->pkeys[0]); i++) {
> -		char name[8];
> -		snprintf(name, sizeof(name), "%u", i);
> -		file_printf(path, name, "0x%04x\n", ntohs(sc->pkeys[i]));
> +		*ports_path_end = '\0';
>  	}
> -	*str = '\0';
>  
>  	/* /sys/class/infiniband_mad/umad0/ */
>  	snprintf(path, sizeof(path), "%s/umad%u", sysfs_infiniband_mad_dir,
> @@ -564,8 +583,7 @@ static struct umad2sim_dev *umad2sim_dev_create(unsigned num, const char *name)
>  	if (sim_client_init(&dev->sim_client) < 0)
>  		goto _error;
>  
> -	dev->port = mad_get_field(&dev->sim_client.portinfo, 0,
> -				  IB_PORT_LOCAL_PORT_F);
> +	dev->port = dev->sim_client.portnum;
>  	for (i = 0; i < arrsize(dev->agents); i++)
>  		dev->agents[i].id = (uint32_t)(-1);
>  	for (i = 0; i < arrsize(dev->agent_idx); i++)
> -- 
> 1.5.6.4
> 



More information about the general mailing list