***SPAM*** Re: [ofa-general] Re: [PATCH] ibsim: Add better end port simulation support
Hal Rosenstock
hal.rosenstock at gmail.com
Tue Feb 17 13:28:40 PST 2009
Sasha,
On Tue, Feb 17, 2009 at 4:18 PM, Sasha Khapyorsky <sashak at voltaire.com> wrote:
> Hi Hal,
>
> On 15:37 Sat 14 Feb , hnrose at comcast.net wrote:
>>
>> Add SIM_PORT environment variable to allow for end port selection
>
> How this would handle case when SIM_PORT=N, but program tries to work
> via another port (for example: SIM_PORT=2 and ibnetdiscover -P 1)?
That's a configuration error. SIM_PORT needs to be set to same port as
program intends to use.
> IOW should port number selection be initiated natively by program rather
> than by using environment variables?
That would've been nice but AFAIT the simulation layer needs the port
number earlier than the program can supply it. Maybe that could be
changed but I didn't dig into that.
>> Signed-off-by: Hal Rosenstock <hal.rosenstock at gmail.com>
>> ---
>> ibsim/ibsim.c | 6 +-
>> include/ibsim.h | 2 +
>> umad2sim/sim_client.c | 49 +++++++++-
>> umad2sim/sim_client.h | 4 +-
>> umad2sim/umad2sim.c | 254 ++++++++++++++++++++++++++-----------------------
>> 5 files changed, 189 insertions(+), 126 deletions(-)
>>
>> diff --git a/ibsim/ibsim.c b/ibsim/ibsim.c
>> index f48e1f0..6a35fdc 100644
>> --- a/ibsim/ibsim.c
>> +++ b/ibsim/ibsim.c
>> @@ -1,5 +1,6 @@
>> /*
>> * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
>> + * Copyright (c) 2009 HNR Consulting. All rights reserved.
>> *
>> * This file is part of ibsim.
>> *
>> @@ -187,7 +188,8 @@ static int sm_exists(Node * node)
>> return 0;
>> }
>>
>> -static int sim_ctl_new_client(Client * cl, struct sim_ctl * ctl, union name_t *from)
>> +static int sim_ctl_new_client(Client * cl, struct sim_ctl * ctl,
>> + union name_t *from)
>> {
>> union name_t name;
>> size_t size;
>> @@ -219,7 +221,7 @@ static int sim_ctl_new_client(Client * cl, struct sim_ctl * ctl, union name_t *f
>> ctl->type = SIM_CTL_ERROR;
>> return -1;
>> }
>> - cl->port = node_get_port(node, 0);
>> + cl->port = node_get_port(node, scl->portnum);
>> VERB("Attaching client %d at node \"%s\" port 0x%" PRIx64,
>> i, node->nodeid, cl->port->portguid);
>> } else {
>> diff --git a/include/ibsim.h b/include/ibsim.h
>> index 15fc37c..66ba6f9 100644
>> --- a/include/ibsim.h
>> +++ b/include/ibsim.h
>> @@ -1,5 +1,6 @@
>> /*
>> * Copyright (c) 2006-2008 Voltaire, Inc. All rights reserved.
>> + * Copyright (c) 2009 HNR Consulting. All rights reserved.
>> *
>> * This file is part of ibsim.
>> *
>> @@ -100,6 +101,7 @@ struct sim_client_info {
>> uint32_t qp;
>> uint32_t issm; /* accept request for qp 0 & 1 */
>> char nodeid[32];
>> + uint32_t portnum;
>> };
>>
>> union name_t {
>> diff --git a/umad2sim/sim_client.c b/umad2sim/sim_client.c
>> index 06bb7a8..1c35109 100644
>> --- a/umad2sim/sim_client.c
>> +++ b/umad2sim/sim_client.c
>> @@ -1,5 +1,6 @@
>> /*
>> * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
>> + * Copyright (c) 2009 HNR Consulting. All rights reserved.
>> *
>> * This file is part of ibsim.
>> *
>> @@ -182,6 +183,7 @@ static int sim_connect(struct sim_client *sc, int id, int qp, char *nodeid)
>> info.id = id;
>> info.issm = 0;
>> info.qp = qp;
>> + info.portnum = sc->portnum;
>>
>> if (nodeid)
>> strncpy(info.nodeid, nodeid, sizeof(info.nodeid) - 1);
>> @@ -202,7 +204,7 @@ static int sim_disconnect(struct sim_client *sc)
>> return sim_ctl(sc, SIM_CTL_DISCONNECT, 0, 0);
>> }
>>
>> -static int sim_init(struct sim_client *sc, char *nodeid)
>> +static int sim_init(struct sim_client *sc, char *nodeid, int portnum)
>> {
>> union name_t name;
>> socklen_t size;
>> @@ -238,6 +240,7 @@ static int sim_init(struct sim_client *sc, char *nodeid)
>> DEBUG("init %d: opened ctl fd %d as \'%s\'",
>> pid, ctlfd, get_name(&name));
>>
>> + sc->portnum = portnum;
>> port = connect_port ? atoi(connect_port) : IBSIM_DEFAULT_SERVER_PORT;
>> size = make_name(&name, connect_host, port, "%s:ctl", socket_basename);
>>
>> @@ -286,9 +289,17 @@ int sim_client_set_sm(struct sim_client *sc, unsigned issm)
>> int sim_client_init(struct sim_client *sc)
>> {
>> char *nodeid;
>> + char *portno;
>> + int i, j = 0, portnum = 0, startport = 1, endport;
>> + uint8_t numports, nodetype;
>> + uint8_t *portinfo;
>>
>> nodeid = getenv("SIM_HOST");
>> - if (sim_init(sc, nodeid) < 0)
>> + portno = getenv("SIM_PORT");
>> + if (portno)
>> + portnum = atoi(portno);
>> +
>> + if (sim_init(sc, nodeid, portnum) < 0)
>> return -1;
>> if (sim_ctl(sc, SIM_CTL_GET_VENDOR, &sc->vendor,
>> sizeof(sc->vendor)) < 0)
>> @@ -296,11 +307,37 @@ int sim_client_init(struct sim_client *sc)
>> if (sim_ctl(sc, SIM_CTL_GET_NODEINFO, sc->nodeinfo,
>> sizeof(sc->nodeinfo)) < 0)
>> goto _exit;
>> + numports = mad_get_field(sc->nodeinfo, 0, IB_NODE_NPORTS_F);
>> + nodetype = mad_get_field(sc->nodeinfo, 0, IB_NODE_TYPE_F);
>> + if (nodetype == 2) { // switch
>> + startport = 0;
>> + endport = 0;
>> + } else {
>> + if (portnum == 0) {
>> + IBWARN("portnum 0 is not valid end port on non switch node");
>> + goto _exit;
>> + }
>
> This makes exporting SIM_PORT environment variable to be mandatory,
> which doesn't look like a good idea for me (personally I will need to
> rewrite some amount of my scripts).
>
> I think that SIM_HOST should be optional and the default behavior
> should be preserved.
>
>> + endport = numports;
>> + }
>> + if (portnum > endport) {
>> + IBWARN("portnum %d is not a valid end port number (%d)",
>> + portnum, endport);
>> + goto _exit;
>> + }
>>
>> - sc->portinfo[0] = 0; // portno requested
>> - if (sim_ctl(sc, SIM_CTL_GET_PORTINFO, sc->portinfo,
>> - sizeof(sc->portinfo)) < 0)
>> + sc->portinfo = malloc(64 * (nodetype != 2 ? numports + 1 : 1)); // portinfo size x number of ports starting at 0
>> + if (!sc->portinfo)
>> goto _exit;
>> +
>> + // loop through end ports
>> + for (i = startport; i <= endport ; i++, j++) {
>> + portinfo = sc->portinfo + 64 * j;
>
> You don't need 'j' - just move portinfo pointer.
OK.
>> + *portinfo = i + 1; // portno requested
>> + if (sim_ctl(sc, SIM_CTL_GET_PORTINFO, portinfo, 64) < 0)
>> + goto _exit;
>> + }
>> +
>> + // although pkeys also per port, current config same on all end ports
>
> Which is not correct really.
What are you referring to ? Is there some config for end port pkeys in
the simulator ?
-- Hal
> Sasha
>
>> if (sim_ctl(sc, SIM_CTL_GET_PKEYS, sc->pkeys, sizeof(sc->pkeys)) < 0)
>> goto _exit;
>> if (getenv("SIM_SET_ISSM"))
>> @@ -315,5 +352,7 @@ int sim_client_init(struct sim_client *sc)
>> void sim_client_exit(struct sim_client *sc)
>> {
>> sim_disconnect(sc);
>> + if (sc->portinfo)
>> + free(sc->portinfo);
>> sc->fd_ctl = sc->fd_pktin = sc->fd_pktout = -1;
>> }
>> diff --git a/umad2sim/sim_client.h b/umad2sim/sim_client.h
>> index 80ed442..0faca80 100644
>> --- a/umad2sim/sim_client.h
>> +++ b/umad2sim/sim_client.h
>> @@ -1,5 +1,6 @@
>> /*
>> * Copyright (c) 2006,2007 Voltaire, Inc. All rights reserved.
>> + * Copyright (c) 2009 HNR Consulting. All rights reserved.
>> *
>> * This file is part of ibsim.
>> *
>> @@ -41,8 +42,9 @@ struct sim_client {
>> int clientid;
>> int fd_pktin, fd_pktout, fd_ctl;
>> struct sim_vendor vendor;
>> + int portnum;
>> uint8_t nodeinfo[64];
>> - uint8_t portinfo[64];
>> + uint8_t *portinfo;
>> uint16_t pkeys[SIM_CTL_MAX_DATA/sizeof(uint16_t)];
>> };
>>
>> diff --git a/umad2sim/umad2sim.c b/umad2sim/umad2sim.c
>> index 8d83a24..6e3c269 100644
>> --- a/umad2sim/umad2sim.c
>> +++ b/umad2sim/umad2sim.c
>> @@ -1,5 +1,6 @@
>> /*
>> * Copyright (c) 2006-2008 Voltaire, Inc. All rights reserved.
>> + * Copyright (c) 2009 HNR Consulting. All rights reserved.
>> *
>> * This file is part of ibsim.
>> *
>> @@ -179,7 +180,10 @@ static int dev_sysfs_create(struct umad2sim_dev *dev)
>> struct sim_client *sc = &dev->sim_client;
>> char *str;
>> uint8_t *portinfo;
>> - int i;
>> + char *ports_path_end;
>> + int i, j;
>> + int startport = 1, endport;
>> + uint8_t numports, nodetype;
>>
>> /* /sys/class/infiniband_mad/abi_version */
>> snprintf(path, sizeof(path), "%s", sysfs_infiniband_mad_dir);
>> @@ -232,123 +236,138 @@ static int dev_sysfs_create(struct umad2sim_dev *dev)
>> strncat(path, "/ports", sizeof(path) - 1);
>> make_path(path);
>>
>> - portinfo = sc->portinfo;
>> -
>> - /* /sys/class/infiniband/mthca0/ports/1/ */
>> - val = mad_get_field(portinfo, 0, IB_PORT_LOCAL_PORT_F);
>> - snprintf(path + strlen(path), sizeof(path) - strlen(path), "/%u", val);
>> - make_path(path);
>> -
>> - /* /sys/class/infiniband/mthca0/ports/1/lid_mask_count */
>> - val = mad_get_field(portinfo, 0, IB_PORT_LMC_F);
>> - file_printf(path, SYS_PORT_LMC, "%d", val);
>> -
>> - /* /sys/class/infiniband/mthca0/ports/1/sm_lid */
>> - val = mad_get_field(portinfo, 0, IB_PORT_SMLID_F);
>> - file_printf(path, SYS_PORT_SMLID, "0x%x", val);
>> -
>> - /* /sys/class/infiniband/mthca0/ports/1/sm_sl */
>> - val = mad_get_field(portinfo, 0, IB_PORT_SMSL_F);
>> - file_printf(path, SYS_PORT_SMSL, "%d", val);
>> -
>> - /* /sys/class/infiniband/mthca0/ports/1/lid */
>> - val = mad_get_field(portinfo, 0, IB_PORT_LID_F);
>> - file_printf(path, SYS_PORT_LID, "0x%x", val);
>> -
>> - /* /sys/class/infiniband/mthca0/ports/1/state */
>> - val = mad_get_field(portinfo, 0, IB_PORT_STATE_F);
>> - if (val == 0)
>> - str = "NOP";
>> - else if (val == 1)
>> - str = "DOWN";
>> - else if (val == 2)
>> - str = "INIT";
>> - else if (val == 3)
>> - str = "ARMED";
>> - else if (val == 4)
>> - str = "ACTIVE";
>> - else if (val == 5)
>> - str = "ACTIVE_DEFER";
>> - else
>> - str = "<unknown>";
>> - file_printf(path, SYS_PORT_STATE, "%d: %s\n", val, str);
>> -
>> - /* /sys/class/infiniband/mthca0/ports/1/phys_state */
>> - val = mad_get_field(portinfo, 0, IB_PORT_PHYS_STATE_F);
>> - if (val == 1)
>> - str = "Sleep";
>> - else if (val == 2)
>> - str = "Polling";
>> - else if (val == 3)
>> - str = "Disabled";
>> - else if (val == 4)
>> - str = "PortConfigurationTraining";
>> - else if (val == 5)
>> - str = "LinkUp";
>> - else if (val == 6)
>> - str = "LinkErrorRecovery";
>> - else if (val == 7)
>> - str = "Phy Test";
>> - else
>> - str = "<unknown>";
>> - file_printf(path, SYS_PORT_PHY_STATE, "%d: %s\n", val, str);
>> -
>> - /* /sys/class/infiniband/mthca0/ports/1/rate */
>> - val = mad_get_field(portinfo, 0, IB_PORT_LINK_WIDTH_ACTIVE_F);
>> - speed = mad_get_field(portinfo, 0, IB_PORT_LINK_SPEED_ACTIVE_F);
>> - if (val == 1)
>> - val = 1;
>> - else if (val == 2)
>> - val = 4;
>> - else if (val == 4)
>> - val = 8;
>> - else if (val == 8)
>> - val = 12;
>> - else
>> - val = 0;
>> - if (speed == 2)
>> - str = " DDR";
>> - else if (speed == 4)
>> - str = " QDR";
>> - else
>> - str = "";
>> - file_printf(path, SYS_PORT_RATE, "%d%s Gb/sec (%dX%s)\n",
>> - (val * speed * 25) / 10,
>> - (val * speed * 25) % 10 ? ".5" : "", val, str);
>> -
>> - /* /sys/class/infiniband/mthca0/ports/1/cap_mask */
>> - val = mad_get_field(portinfo, 0, IB_PORT_CAPMASK_F);
>> - file_printf(path, SYS_PORT_CAPMASK, "0x%08x", val);
>> -
>> - /* /sys/class/infiniband/mthca0/ports/1/gids/0 */
>> - str = path + strlen(path);
>> - strncat(path, "/gids", sizeof(path) - 1);
>> - make_path(path);
>> - *str = '\0';
>> - gid = mad_get_field64(portinfo, 0, IB_PORT_GID_PREFIX_F);
>> - guid = mad_get_field64(sc->nodeinfo, 0, IB_NODE_GUID_F) +
>> - mad_get_field(portinfo, 0, IB_PORT_LOCAL_PORT_F);
>> - file_printf(path, SYS_PORT_GID,
>> - "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n",
>> - (uint16_t) ((gid >> 48) & 0xffff),
>> - (uint16_t) ((gid >> 32) & 0xffff),
>> - (uint16_t) ((gid >> 16) & 0xffff),
>> - (uint16_t) ((gid >> 0) & 0xffff),
>> - (uint16_t) ((guid >> 48) & 0xffff),
>> - (uint16_t) ((guid >> 32) & 0xffff),
>> - (uint16_t) ((guid >> 16) & 0xffff),
>> - (uint16_t) ((guid >> 0) & 0xffff));
>> + numports = mad_get_field(sc->nodeinfo, 0, IB_NODE_NPORTS_F);
>> + nodetype = mad_get_field(sc->nodeinfo, 0, IB_NODE_TYPE_F);
>> + if (nodetype == 2) { // switch
>> + startport = 0;
>> + endport = 0;
>> + } else
>> + endport = numports;
>> +
>> + ports_path_end = path + strlen(path);
>> +
>> + // loop through end ports
>> + for (j = startport; j <= endport; j++) {
>> +
>> + portinfo = sc->portinfo + 64 * j;
>> +
>> + /* /sys/class/infiniband/mthca0/ports/<n>/ */
>> + val = mad_get_field(portinfo, 0, IB_PORT_LOCAL_PORT_F);
>> + snprintf(path + strlen(path), sizeof(path) - strlen(path), "/%u", val);
>> + make_path(path);
>> +
>> + /* /sys/class/infiniband/mthca0/ports/<n>/lid_mask_count */
>> + val = mad_get_field(portinfo, 0, IB_PORT_LMC_F);
>> + file_printf(path, SYS_PORT_LMC, "%d", val);
>> +
>> + /* /sys/class/infiniband/mthca0/ports/<n>/sm_lid */
>> + val = mad_get_field(portinfo, 0, IB_PORT_SMLID_F);
>> + file_printf(path, SYS_PORT_SMLID, "0x%x", val);
>> +
>> + /* /sys/class/infiniband/mthca0/ports/<n>/sm_sl */
>> + val = mad_get_field(portinfo, 0, IB_PORT_SMSL_F);
>> + file_printf(path, SYS_PORT_SMSL, "%d", val);
>> +
>> + /* /sys/class/infiniband/mthca0/ports/<n>/lid */
>> + val = mad_get_field(portinfo, 0, IB_PORT_LID_F);
>> + file_printf(path, SYS_PORT_LID, "0x%x", val);
>> +
>> + /* /sys/class/infiniband/mthca0/ports/<n>/state */
>> + val = mad_get_field(portinfo, 0, IB_PORT_STATE_F);
>> + if (val == 0)
>> + str = "NOP";
>> + else if (val == 1)
>> + str = "DOWN";
>> + else if (val == 2)
>> + str = "INIT";
>> + else if (val == 3)
>> + str = "ARMED";
>> + else if (val == 4)
>> + str = "ACTIVE";
>> + else if (val == 5)
>> + str = "ACTIVE_DEFER";
>> + else
>> + str = "<unknown>";
>> + file_printf(path, SYS_PORT_STATE, "%d: %s\n", val, str);
>> +
>> + /* /sys/class/infiniband/mthca0/ports/<n>/phys_state */
>> + val = mad_get_field(portinfo, 0, IB_PORT_PHYS_STATE_F);
>> + if (val == 1)
>> + str = "Sleep";
>> + else if (val == 2)
>> + str = "Polling";
>> + else if (val == 3)
>> + str = "Disabled";
>> + else if (val == 4)
>> + str = "PortConfigurationTraining";
>> + else if (val == 5)
>> + str = "LinkUp";
>> + else if (val == 6)
>> + str = "LinkErrorRecovery";
>> + else if (val == 7)
>> + str = "Phy Test";
>> + else
>> + str = "<unknown>";
>> + file_printf(path, SYS_PORT_PHY_STATE, "%d: %s\n", val, str);
>> +
>> + /* /sys/class/infiniband/mthca0/ports/<n>/rate */
>> + val = mad_get_field(portinfo, 0, IB_PORT_LINK_WIDTH_ACTIVE_F);
>> + speed = mad_get_field(portinfo, 0, IB_PORT_LINK_SPEED_ACTIVE_F);
>> + if (val == 1)
>> + val = 1;
>> + else if (val == 2)
>> + val = 4;
>> + else if (val == 4)
>> + val = 8;
>> + else if (val == 8)
>> + val = 12;
>> + else
>> + val = 0;
>> + if (speed == 2)
>> + str = " DDR";
>> + else if (speed == 4)
>> + str = " QDR";
>> + else
>> + str = "";
>> + file_printf(path, SYS_PORT_RATE, "%d%s Gb/sec (%dX%s)\n",
>> + (val * speed * 25) / 10,
>> + (val * speed * 25) % 10 ? ".5" : "", val, str);
>> +
>> + /* /sys/class/infiniband/mthca0/ports/<n>/cap_mask */
>> + val = mad_get_field(portinfo, 0, IB_PORT_CAPMASK_F);
>> + file_printf(path, SYS_PORT_CAPMASK, "0x%08x", val);
>> +
>> + /* /sys/class/infiniband/mthca0/ports/<n>/gids/0 */
>> + str = path + strlen(path);
>> + strncat(path, "/gids", sizeof(path) - 1);
>> + make_path(path);
>> + *str = '\0';
>> + gid = mad_get_field64(portinfo, 0, IB_PORT_GID_PREFIX_F);
>> + guid = mad_get_field64(sc->nodeinfo, 0, IB_NODE_GUID_F) + j;
>> + file_printf(path, SYS_PORT_GID,
>> + "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n",
>> + (uint16_t) ((gid >> 48) & 0xffff),
>> + (uint16_t) ((gid >> 32) & 0xffff),
>> + (uint16_t) ((gid >> 16) & 0xffff),
>> + (uint16_t) ((gid >> 0) & 0xffff),
>> + (uint16_t) ((guid >> 48) & 0xffff),
>> + (uint16_t) ((guid >> 32) & 0xffff),
>> + (uint16_t) ((guid >> 16) & 0xffff),
>> + (uint16_t) ((guid >> 0) & 0xffff));
>> +
>> + /* /sys/class/infiniband/mthca0/ports/<n>/pkeys/0 */
>> + str = path + strlen(path);
>> + strncat(path, "/pkeys", sizeof(path) - 1);
>> + make_path(path);
>> + for (i = 0; i < sizeof(sc->pkeys)/sizeof(sc->pkeys[0]); i++) {
>> + char name[8];
>> + snprintf(name, sizeof(name), "%u", i);
>> + file_printf(path, name, "0x%04x\n", ntohs(sc->pkeys[i]));
>> + }
>> + *str = '\0';
>>
>> - /* /sys/class/infiniband/mthca0/ports/1/pkeys/0 */
>> - str = path + strlen(path);
>> - strncat(path, "/pkeys", sizeof(path) - 1);
>> - make_path(path);
>> - for (i = 0; i < sizeof(sc->pkeys)/sizeof(sc->pkeys[0]); i++) {
>> - char name[8];
>> - snprintf(name, sizeof(name), "%u", i);
>> - file_printf(path, name, "0x%04x\n", ntohs(sc->pkeys[i]));
>> + *ports_path_end = '\0';
>> }
>> - *str = '\0';
>>
>> /* /sys/class/infiniband_mad/umad0/ */
>> snprintf(path, sizeof(path), "%s/umad%u", sysfs_infiniband_mad_dir,
>> @@ -564,8 +583,7 @@ static struct umad2sim_dev *umad2sim_dev_create(unsigned num, const char *name)
>> if (sim_client_init(&dev->sim_client) < 0)
>> goto _error;
>>
>> - dev->port = mad_get_field(&dev->sim_client.portinfo, 0,
>> - IB_PORT_LOCAL_PORT_F);
>> + dev->port = dev->sim_client.portnum;
>> for (i = 0; i < arrsize(dev->agents); i++)
>> dev->agents[i].id = (uint32_t)(-1);
>> for (i = 0; i < arrsize(dev->agent_idx); i++)
>> --
>> 1.5.6.4
>>
> _______________________________________________
> general mailing list
> general at lists.openfabrics.org
> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/general
>
> To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
>
More information about the general
mailing list