[ofa-general] Re: [PATCH] ibsim: Add better end port simulation support
Sasha Khapyorsky
sashak at voltaire.com
Tue Feb 17 13:18:48 PST 2009
Hi Hal,
On 15:37 Sat 14 Feb , hnrose at comcast.net wrote:
>
> Add SIM_PORT environment variable to allow for end port selection
How this would handle case when SIM_PORT=N, but program tries to work
via another port (for example: SIM_PORT=2 and ibnetdiscover -P 1)?
IOW should port number selection be initiated natively by program rather
than by using environment variables?
> Signed-off-by: Hal Rosenstock <hal.rosenstock at gmail.com>
> ---
> ibsim/ibsim.c | 6 +-
> include/ibsim.h | 2 +
> umad2sim/sim_client.c | 49 +++++++++-
> umad2sim/sim_client.h | 4 +-
> umad2sim/umad2sim.c | 254 ++++++++++++++++++++++++++-----------------------
> 5 files changed, 189 insertions(+), 126 deletions(-)
>
> diff --git a/ibsim/ibsim.c b/ibsim/ibsim.c
> index f48e1f0..6a35fdc 100644
> --- a/ibsim/ibsim.c
> +++ b/ibsim/ibsim.c
> @@ -1,5 +1,6 @@
> /*
> * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
> + * Copyright (c) 2009 HNR Consulting. All rights reserved.
> *
> * This file is part of ibsim.
> *
> @@ -187,7 +188,8 @@ static int sm_exists(Node * node)
> return 0;
> }
>
> -static int sim_ctl_new_client(Client * cl, struct sim_ctl * ctl, union name_t *from)
> +static int sim_ctl_new_client(Client * cl, struct sim_ctl * ctl,
> + union name_t *from)
> {
> union name_t name;
> size_t size;
> @@ -219,7 +221,7 @@ static int sim_ctl_new_client(Client * cl, struct sim_ctl * ctl, union name_t *f
> ctl->type = SIM_CTL_ERROR;
> return -1;
> }
> - cl->port = node_get_port(node, 0);
> + cl->port = node_get_port(node, scl->portnum);
> VERB("Attaching client %d at node \"%s\" port 0x%" PRIx64,
> i, node->nodeid, cl->port->portguid);
> } else {
> diff --git a/include/ibsim.h b/include/ibsim.h
> index 15fc37c..66ba6f9 100644
> --- a/include/ibsim.h
> +++ b/include/ibsim.h
> @@ -1,5 +1,6 @@
> /*
> * Copyright (c) 2006-2008 Voltaire, Inc. All rights reserved.
> + * Copyright (c) 2009 HNR Consulting. All rights reserved.
> *
> * This file is part of ibsim.
> *
> @@ -100,6 +101,7 @@ struct sim_client_info {
> uint32_t qp;
> uint32_t issm; /* accept request for qp 0 & 1 */
> char nodeid[32];
> + uint32_t portnum;
> };
>
> union name_t {
> diff --git a/umad2sim/sim_client.c b/umad2sim/sim_client.c
> index 06bb7a8..1c35109 100644
> --- a/umad2sim/sim_client.c
> +++ b/umad2sim/sim_client.c
> @@ -1,5 +1,6 @@
> /*
> * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
> + * Copyright (c) 2009 HNR Consulting. All rights reserved.
> *
> * This file is part of ibsim.
> *
> @@ -182,6 +183,7 @@ static int sim_connect(struct sim_client *sc, int id, int qp, char *nodeid)
> info.id = id;
> info.issm = 0;
> info.qp = qp;
> + info.portnum = sc->portnum;
>
> if (nodeid)
> strncpy(info.nodeid, nodeid, sizeof(info.nodeid) - 1);
> @@ -202,7 +204,7 @@ static int sim_disconnect(struct sim_client *sc)
> return sim_ctl(sc, SIM_CTL_DISCONNECT, 0, 0);
> }
>
> -static int sim_init(struct sim_client *sc, char *nodeid)
> +static int sim_init(struct sim_client *sc, char *nodeid, int portnum)
> {
> union name_t name;
> socklen_t size;
> @@ -238,6 +240,7 @@ static int sim_init(struct sim_client *sc, char *nodeid)
> DEBUG("init %d: opened ctl fd %d as \'%s\'",
> pid, ctlfd, get_name(&name));
>
> + sc->portnum = portnum;
> port = connect_port ? atoi(connect_port) : IBSIM_DEFAULT_SERVER_PORT;
> size = make_name(&name, connect_host, port, "%s:ctl", socket_basename);
>
> @@ -286,9 +289,17 @@ int sim_client_set_sm(struct sim_client *sc, unsigned issm)
> int sim_client_init(struct sim_client *sc)
> {
> char *nodeid;
> + char *portno;
> + int i, j = 0, portnum = 0, startport = 1, endport;
> + uint8_t numports, nodetype;
> + uint8_t *portinfo;
>
> nodeid = getenv("SIM_HOST");
> - if (sim_init(sc, nodeid) < 0)
> + portno = getenv("SIM_PORT");
> + if (portno)
> + portnum = atoi(portno);
> +
> + if (sim_init(sc, nodeid, portnum) < 0)
> return -1;
> if (sim_ctl(sc, SIM_CTL_GET_VENDOR, &sc->vendor,
> sizeof(sc->vendor)) < 0)
> @@ -296,11 +307,37 @@ int sim_client_init(struct sim_client *sc)
> if (sim_ctl(sc, SIM_CTL_GET_NODEINFO, sc->nodeinfo,
> sizeof(sc->nodeinfo)) < 0)
> goto _exit;
> + numports = mad_get_field(sc->nodeinfo, 0, IB_NODE_NPORTS_F);
> + nodetype = mad_get_field(sc->nodeinfo, 0, IB_NODE_TYPE_F);
> + if (nodetype == 2) { // switch
> + startport = 0;
> + endport = 0;
> + } else {
> + if (portnum == 0) {
> + IBWARN("portnum 0 is not valid end port on non switch node");
> + goto _exit;
> + }
This makes exporting SIM_PORT environment variable to be mandatory,
which doesn't look like a good idea for me (personally I will need to
rewrite some amount of my scripts).
I think that SIM_HOST should be optional and the default behavior
should be preserved.
> + endport = numports;
> + }
> + if (portnum > endport) {
> + IBWARN("portnum %d is not a valid end port number (%d)",
> + portnum, endport);
> + goto _exit;
> + }
>
> - sc->portinfo[0] = 0; // portno requested
> - if (sim_ctl(sc, SIM_CTL_GET_PORTINFO, sc->portinfo,
> - sizeof(sc->portinfo)) < 0)
> + sc->portinfo = malloc(64 * (nodetype != 2 ? numports + 1 : 1)); // portinfo size x number of ports starting at 0
> + if (!sc->portinfo)
> goto _exit;
> +
> + // loop through end ports
> + for (i = startport; i <= endport ; i++, j++) {
> + portinfo = sc->portinfo + 64 * j;
You don't need 'j' - just move portinfo pointer.
> + *portinfo = i + 1; // portno requested
> + if (sim_ctl(sc, SIM_CTL_GET_PORTINFO, portinfo, 64) < 0)
> + goto _exit;
> + }
> +
> + // although pkeys also per port, current config same on all end ports
Which is not correct really.
Sasha
> if (sim_ctl(sc, SIM_CTL_GET_PKEYS, sc->pkeys, sizeof(sc->pkeys)) < 0)
> goto _exit;
> if (getenv("SIM_SET_ISSM"))
> @@ -315,5 +352,7 @@ int sim_client_init(struct sim_client *sc)
> void sim_client_exit(struct sim_client *sc)
> {
> sim_disconnect(sc);
> + if (sc->portinfo)
> + free(sc->portinfo);
> sc->fd_ctl = sc->fd_pktin = sc->fd_pktout = -1;
> }
> diff --git a/umad2sim/sim_client.h b/umad2sim/sim_client.h
> index 80ed442..0faca80 100644
> --- a/umad2sim/sim_client.h
> +++ b/umad2sim/sim_client.h
> @@ -1,5 +1,6 @@
> /*
> * Copyright (c) 2006,2007 Voltaire, Inc. All rights reserved.
> + * Copyright (c) 2009 HNR Consulting. All rights reserved.
> *
> * This file is part of ibsim.
> *
> @@ -41,8 +42,9 @@ struct sim_client {
> int clientid;
> int fd_pktin, fd_pktout, fd_ctl;
> struct sim_vendor vendor;
> + int portnum;
> uint8_t nodeinfo[64];
> - uint8_t portinfo[64];
> + uint8_t *portinfo;
> uint16_t pkeys[SIM_CTL_MAX_DATA/sizeof(uint16_t)];
> };
>
> diff --git a/umad2sim/umad2sim.c b/umad2sim/umad2sim.c
> index 8d83a24..6e3c269 100644
> --- a/umad2sim/umad2sim.c
> +++ b/umad2sim/umad2sim.c
> @@ -1,5 +1,6 @@
> /*
> * Copyright (c) 2006-2008 Voltaire, Inc. All rights reserved.
> + * Copyright (c) 2009 HNR Consulting. All rights reserved.
> *
> * This file is part of ibsim.
> *
> @@ -179,7 +180,10 @@ static int dev_sysfs_create(struct umad2sim_dev *dev)
> struct sim_client *sc = &dev->sim_client;
> char *str;
> uint8_t *portinfo;
> - int i;
> + char *ports_path_end;
> + int i, j;
> + int startport = 1, endport;
> + uint8_t numports, nodetype;
>
> /* /sys/class/infiniband_mad/abi_version */
> snprintf(path, sizeof(path), "%s", sysfs_infiniband_mad_dir);
> @@ -232,123 +236,138 @@ static int dev_sysfs_create(struct umad2sim_dev *dev)
> strncat(path, "/ports", sizeof(path) - 1);
> make_path(path);
>
> - portinfo = sc->portinfo;
> -
> - /* /sys/class/infiniband/mthca0/ports/1/ */
> - val = mad_get_field(portinfo, 0, IB_PORT_LOCAL_PORT_F);
> - snprintf(path + strlen(path), sizeof(path) - strlen(path), "/%u", val);
> - make_path(path);
> -
> - /* /sys/class/infiniband/mthca0/ports/1/lid_mask_count */
> - val = mad_get_field(portinfo, 0, IB_PORT_LMC_F);
> - file_printf(path, SYS_PORT_LMC, "%d", val);
> -
> - /* /sys/class/infiniband/mthca0/ports/1/sm_lid */
> - val = mad_get_field(portinfo, 0, IB_PORT_SMLID_F);
> - file_printf(path, SYS_PORT_SMLID, "0x%x", val);
> -
> - /* /sys/class/infiniband/mthca0/ports/1/sm_sl */
> - val = mad_get_field(portinfo, 0, IB_PORT_SMSL_F);
> - file_printf(path, SYS_PORT_SMSL, "%d", val);
> -
> - /* /sys/class/infiniband/mthca0/ports/1/lid */
> - val = mad_get_field(portinfo, 0, IB_PORT_LID_F);
> - file_printf(path, SYS_PORT_LID, "0x%x", val);
> -
> - /* /sys/class/infiniband/mthca0/ports/1/state */
> - val = mad_get_field(portinfo, 0, IB_PORT_STATE_F);
> - if (val == 0)
> - str = "NOP";
> - else if (val == 1)
> - str = "DOWN";
> - else if (val == 2)
> - str = "INIT";
> - else if (val == 3)
> - str = "ARMED";
> - else if (val == 4)
> - str = "ACTIVE";
> - else if (val == 5)
> - str = "ACTIVE_DEFER";
> - else
> - str = "<unknown>";
> - file_printf(path, SYS_PORT_STATE, "%d: %s\n", val, str);
> -
> - /* /sys/class/infiniband/mthca0/ports/1/phys_state */
> - val = mad_get_field(portinfo, 0, IB_PORT_PHYS_STATE_F);
> - if (val == 1)
> - str = "Sleep";
> - else if (val == 2)
> - str = "Polling";
> - else if (val == 3)
> - str = "Disabled";
> - else if (val == 4)
> - str = "PortConfigurationTraining";
> - else if (val == 5)
> - str = "LinkUp";
> - else if (val == 6)
> - str = "LinkErrorRecovery";
> - else if (val == 7)
> - str = "Phy Test";
> - else
> - str = "<unknown>";
> - file_printf(path, SYS_PORT_PHY_STATE, "%d: %s\n", val, str);
> -
> - /* /sys/class/infiniband/mthca0/ports/1/rate */
> - val = mad_get_field(portinfo, 0, IB_PORT_LINK_WIDTH_ACTIVE_F);
> - speed = mad_get_field(portinfo, 0, IB_PORT_LINK_SPEED_ACTIVE_F);
> - if (val == 1)
> - val = 1;
> - else if (val == 2)
> - val = 4;
> - else if (val == 4)
> - val = 8;
> - else if (val == 8)
> - val = 12;
> - else
> - val = 0;
> - if (speed == 2)
> - str = " DDR";
> - else if (speed == 4)
> - str = " QDR";
> - else
> - str = "";
> - file_printf(path, SYS_PORT_RATE, "%d%s Gb/sec (%dX%s)\n",
> - (val * speed * 25) / 10,
> - (val * speed * 25) % 10 ? ".5" : "", val, str);
> -
> - /* /sys/class/infiniband/mthca0/ports/1/cap_mask */
> - val = mad_get_field(portinfo, 0, IB_PORT_CAPMASK_F);
> - file_printf(path, SYS_PORT_CAPMASK, "0x%08x", val);
> -
> - /* /sys/class/infiniband/mthca0/ports/1/gids/0 */
> - str = path + strlen(path);
> - strncat(path, "/gids", sizeof(path) - 1);
> - make_path(path);
> - *str = '\0';
> - gid = mad_get_field64(portinfo, 0, IB_PORT_GID_PREFIX_F);
> - guid = mad_get_field64(sc->nodeinfo, 0, IB_NODE_GUID_F) +
> - mad_get_field(portinfo, 0, IB_PORT_LOCAL_PORT_F);
> - file_printf(path, SYS_PORT_GID,
> - "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n",
> - (uint16_t) ((gid >> 48) & 0xffff),
> - (uint16_t) ((gid >> 32) & 0xffff),
> - (uint16_t) ((gid >> 16) & 0xffff),
> - (uint16_t) ((gid >> 0) & 0xffff),
> - (uint16_t) ((guid >> 48) & 0xffff),
> - (uint16_t) ((guid >> 32) & 0xffff),
> - (uint16_t) ((guid >> 16) & 0xffff),
> - (uint16_t) ((guid >> 0) & 0xffff));
> + numports = mad_get_field(sc->nodeinfo, 0, IB_NODE_NPORTS_F);
> + nodetype = mad_get_field(sc->nodeinfo, 0, IB_NODE_TYPE_F);
> + if (nodetype == 2) { // switch
> + startport = 0;
> + endport = 0;
> + } else
> + endport = numports;
> +
> + ports_path_end = path + strlen(path);
> +
> + // loop through end ports
> + for (j = startport; j <= endport; j++) {
> +
> + portinfo = sc->portinfo + 64 * j;
> +
> + /* /sys/class/infiniband/mthca0/ports/<n>/ */
> + val = mad_get_field(portinfo, 0, IB_PORT_LOCAL_PORT_F);
> + snprintf(path + strlen(path), sizeof(path) - strlen(path), "/%u", val);
> + make_path(path);
> +
> + /* /sys/class/infiniband/mthca0/ports/<n>/lid_mask_count */
> + val = mad_get_field(portinfo, 0, IB_PORT_LMC_F);
> + file_printf(path, SYS_PORT_LMC, "%d", val);
> +
> + /* /sys/class/infiniband/mthca0/ports/<n>/sm_lid */
> + val = mad_get_field(portinfo, 0, IB_PORT_SMLID_F);
> + file_printf(path, SYS_PORT_SMLID, "0x%x", val);
> +
> + /* /sys/class/infiniband/mthca0/ports/<n>/sm_sl */
> + val = mad_get_field(portinfo, 0, IB_PORT_SMSL_F);
> + file_printf(path, SYS_PORT_SMSL, "%d", val);
> +
> + /* /sys/class/infiniband/mthca0/ports/<n>/lid */
> + val = mad_get_field(portinfo, 0, IB_PORT_LID_F);
> + file_printf(path, SYS_PORT_LID, "0x%x", val);
> +
> + /* /sys/class/infiniband/mthca0/ports/<n>/state */
> + val = mad_get_field(portinfo, 0, IB_PORT_STATE_F);
> + if (val == 0)
> + str = "NOP";
> + else if (val == 1)
> + str = "DOWN";
> + else if (val == 2)
> + str = "INIT";
> + else if (val == 3)
> + str = "ARMED";
> + else if (val == 4)
> + str = "ACTIVE";
> + else if (val == 5)
> + str = "ACTIVE_DEFER";
> + else
> + str = "<unknown>";
> + file_printf(path, SYS_PORT_STATE, "%d: %s\n", val, str);
> +
> + /* /sys/class/infiniband/mthca0/ports/<n>/phys_state */
> + val = mad_get_field(portinfo, 0, IB_PORT_PHYS_STATE_F);
> + if (val == 1)
> + str = "Sleep";
> + else if (val == 2)
> + str = "Polling";
> + else if (val == 3)
> + str = "Disabled";
> + else if (val == 4)
> + str = "PortConfigurationTraining";
> + else if (val == 5)
> + str = "LinkUp";
> + else if (val == 6)
> + str = "LinkErrorRecovery";
> + else if (val == 7)
> + str = "Phy Test";
> + else
> + str = "<unknown>";
> + file_printf(path, SYS_PORT_PHY_STATE, "%d: %s\n", val, str);
> +
> + /* /sys/class/infiniband/mthca0/ports/<n>/rate */
> + val = mad_get_field(portinfo, 0, IB_PORT_LINK_WIDTH_ACTIVE_F);
> + speed = mad_get_field(portinfo, 0, IB_PORT_LINK_SPEED_ACTIVE_F);
> + if (val == 1)
> + val = 1;
> + else if (val == 2)
> + val = 4;
> + else if (val == 4)
> + val = 8;
> + else if (val == 8)
> + val = 12;
> + else
> + val = 0;
> + if (speed == 2)
> + str = " DDR";
> + else if (speed == 4)
> + str = " QDR";
> + else
> + str = "";
> + file_printf(path, SYS_PORT_RATE, "%d%s Gb/sec (%dX%s)\n",
> + (val * speed * 25) / 10,
> + (val * speed * 25) % 10 ? ".5" : "", val, str);
> +
> + /* /sys/class/infiniband/mthca0/ports/<n>/cap_mask */
> + val = mad_get_field(portinfo, 0, IB_PORT_CAPMASK_F);
> + file_printf(path, SYS_PORT_CAPMASK, "0x%08x", val);
> +
> + /* /sys/class/infiniband/mthca0/ports/<n>/gids/0 */
> + str = path + strlen(path);
> + strncat(path, "/gids", sizeof(path) - 1);
> + make_path(path);
> + *str = '\0';
> + gid = mad_get_field64(portinfo, 0, IB_PORT_GID_PREFIX_F);
> + guid = mad_get_field64(sc->nodeinfo, 0, IB_NODE_GUID_F) + j;
> + file_printf(path, SYS_PORT_GID,
> + "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n",
> + (uint16_t) ((gid >> 48) & 0xffff),
> + (uint16_t) ((gid >> 32) & 0xffff),
> + (uint16_t) ((gid >> 16) & 0xffff),
> + (uint16_t) ((gid >> 0) & 0xffff),
> + (uint16_t) ((guid >> 48) & 0xffff),
> + (uint16_t) ((guid >> 32) & 0xffff),
> + (uint16_t) ((guid >> 16) & 0xffff),
> + (uint16_t) ((guid >> 0) & 0xffff));
> +
> + /* /sys/class/infiniband/mthca0/ports/<n>/pkeys/0 */
> + str = path + strlen(path);
> + strncat(path, "/pkeys", sizeof(path) - 1);
> + make_path(path);
> + for (i = 0; i < sizeof(sc->pkeys)/sizeof(sc->pkeys[0]); i++) {
> + char name[8];
> + snprintf(name, sizeof(name), "%u", i);
> + file_printf(path, name, "0x%04x\n", ntohs(sc->pkeys[i]));
> + }
> + *str = '\0';
>
> - /* /sys/class/infiniband/mthca0/ports/1/pkeys/0 */
> - str = path + strlen(path);
> - strncat(path, "/pkeys", sizeof(path) - 1);
> - make_path(path);
> - for (i = 0; i < sizeof(sc->pkeys)/sizeof(sc->pkeys[0]); i++) {
> - char name[8];
> - snprintf(name, sizeof(name), "%u", i);
> - file_printf(path, name, "0x%04x\n", ntohs(sc->pkeys[i]));
> + *ports_path_end = '\0';
> }
> - *str = '\0';
>
> /* /sys/class/infiniband_mad/umad0/ */
> snprintf(path, sizeof(path), "%s/umad%u", sysfs_infiniband_mad_dir,
> @@ -564,8 +583,7 @@ static struct umad2sim_dev *umad2sim_dev_create(unsigned num, const char *name)
> if (sim_client_init(&dev->sim_client) < 0)
> goto _error;
>
> - dev->port = mad_get_field(&dev->sim_client.portinfo, 0,
> - IB_PORT_LOCAL_PORT_F);
> + dev->port = dev->sim_client.portnum;
> for (i = 0; i < arrsize(dev->agents); i++)
> dev->agents[i].id = (uint32_t)(-1);
> for (i = 0; i < arrsize(dev->agent_idx); i++)
> --
> 1.5.6.4
>
More information about the general
mailing list