[ewg] [Q] IPoIB with multiple ports setup

[Richard Croucher]

InfiniBand is really designed to run connected through a switch.  In
which case your topology would work.   OpenSM only uses 1 port.  In your
case it is using port 1 on Server1, hence,  it is ignoring Client2
connected on port2.   Connected via a switch, this would not be a
problem, but without the switch, it is.

I would directly connect your client 1 to client 2, and run OpenSM on
both of them.   OpenSM will then elect one as the master and assign the
other as the slave .   This should then work fine, since both clients
will be discovered.

Richard


On Fri, 2010-11-26 at 11:13 +0900, Hiroyuki Sato wrote:

> Hello members.
> 
> I'm newbie about Infiniband
> 
> Does anybody knows good example configuration about openib and opensmd ?
> 
> I would like to setup IPoIB with multiple ports
> It seems that I have to setup opensmd.
> 
> I read the documentation opensm(8) which is bundled with CentOS
> but I couldn't setup multiple ports.
> 
> I'm trying the following environment
> 
>   server1 <-> client1 : ping OK
>   server1 <-> client2 : ping NG
> 
> Could you tell me what is wrong??
> 
> 
> * Environemnt
>   client1 and client2 belongs in different subnets
> 
>   SubnetManager
>   192.168.100.231(port1)
>   192.168.200.231(port2)        client1(192.168.100.232)
>   +----------+                  +----------+
>   |          | 192.168.100.0/24 |          |
>   |          port1------------port1        |
>   |          port2--+         port2        |
>   |          |      |           |          |
>   +----------+      |           +----------+
>                     |
>                     |
>                     |           client2(192.168.200.233)
>                     |           +----------+
>                     |           |          |
>                     +----------port1       |
>               192.168.200.0/24 port2       |
>                                 |          |
>                                 +----------+
> 
> 
> OS: CentOS 5.5
> Kernel: 2.6.18-194.26.1.el
> OpenSM: 3.3.3 bundled with CentOS
> 
> Thanks in advice
> 
> -- 
> Hiroyuki Sato
> 
> 
>   egrep -v '^(#|$)' /etc/ofed/openib.conf
>   IPOIB_LOAD=yes
>   SDP_LOAD=yes
>   RDS_LOAD=yes
>   SRP_LOAD=yes
>   ISER_LOAD=yes
>   FIXUP_MTRR_REGS=no
> 
> 
>   egrep -v '^(#|$)' /etc/ofed/opensm.conf
> 
>   guid 0x0000000000000000
>   m_key 0x0000000000000000
>   m_key_lease_period 0
>   sm_key 0x0000000000000001
>   sa_key 0x0000000000000001
>   subnet_prefix 0xfe80000000000000
>   lmc 0
>   lmc_esp0 FALSE
>   packet_life_time 0x12
>   vl_stall_count 0x07
>   leaf_vl_stall_count 0x07
>   head_of_queue_lifetime 0x12
>   leaf_head_of_queue_lifetime 0x10
>   max_op_vls 5
>   force_link_speed 15
>   subnet_timeout 18
>   local_phy_errors_threshold 0x08
>   overrun_errors_threshold 0x08
>   partition_config_file /etc/ofed/partitions.conf
>   no_partition_enforcement FALSE
>   sweep_interval 10
>   reassign_lids FALSE
>   force_heavy_sweep FALSE
>   sweep_on_trap TRUE
>   port_profile_switch_nodes FALSE
>   port_prof_ignore_file (null)
>   routing_engine (null)
>   connect_roots FALSE
>   use_ucast_cache FALSE
>   lid_matrix_dump_file (null)
>   lfts_file (null)
>   root_guid_file (null)
>   cn_guid_file (null)
>   ids_guid_file (null)
>   guid_routing_order_file (null)
>   sa_db_file (null)
>   sm_priority 15
>   ignore_other_sm FALSE
>   sminfo_polling_timeout 10000
>   polling_retry_number 4
>   honor_guid2lid_file FALSE
>   max_wire_smps 32
>   transaction_timeout 200
>   max_msg_fifo_timeout 10000
>   single_thread FALSE
>   daemon TRUE
>   sm_inactive FALSE
>   babbling_port_policy FALSE
>   event_plugin_name (null)
>   node_name_map_name (null)
>   log_flags 0x03
>   force_log_flush FALSE
>   log_file /var/log/opensm.log
>   log_max_size 0
>   accum_log_file TRUE
>   dump_files_dir /var/log/
>   enable_quirks FALSE
>   no_clients_rereg FALSE
>   disable_multicast FALSE
>   exit_on_fatal TRUE
>   console off
>   console_port 10000
>   qos FALSE
>   qos_policy_file /etc/ofed/qos-policy.conf
>   qos_max_vls 0
>   qos_high_limit -1
>   qos_vlarb_high (null)
>   qos_vlarb_low (null)
>   qos_sl2vl (null)
>   qos_ca_max_vls 0
>   qos_ca_high_limit -1
>   qos_ca_vlarb_high (null)
>   qos_ca_vlarb_low (null)
>   qos_ca_sl2vl (null)
>   qos_sw0_max_vls 0
>   qos_sw0_high_limit -1
>   qos_sw0_vlarb_high (null)
>   qos_sw0_vlarb_low (null)
>   qos_sw0_sl2vl (null)
>   qos_swe_max_vls 0
>   qos_swe_high_limit -1
>   qos_swe_vlarb_high (null)
>   qos_swe_vlarb_low (null)
>   qos_swe_sl2vl (null)
>   qos_rtr_max_vls 0
>   qos_rtr_high_limit -1
>   qos_rtr_vlarb_high (null)
>   qos_rtr_vlarb_low (null)
>   qos_rtr_sl2vl (null)
>   prefix_routes_file /etc/ofed/prefix-routes.conf
>   consolidate_ipv6_snm_req FALSE
> 
>   # cat /etc/sysconfig/network-scripts/ifcfg-ib0
>   DEVICE=ib0
>   BOOTPROTO=static
>   DHCPCLASS=
>   IPADDR=192.168.100.231
>   NETMASK=255.255.255.0
>   ONBOOT=yes
> 
>   cat /etc/sysconfig/network-scripts/ifcfg-ib1
>   DEVICE=ib1
>   BOOTPROTO=static
>   DHCPCLASS=
>   IPADDR=192.168.200.231
>   NETMASK=255.255.255.0
>   ONBOOT=yes
> 
> client1
> 
>   /sbin/chkconfig oepnibd on
> 
>   cat /etc/sysconfig/network-scripts/ifcfg-ib0
>   DEVICE=ib0
>   BOOTPROTO=static
>   DHCPCLASS=
>   IPADDR=192.168.100.232
>   NETMASK=255.255.255.0
>   ONBOOT=yes
> 
> client2
> 
>   /sbin/chkconfig oepnibd on
> 
>   cat /etc/sysconfig/network-scripts/ifcfg-ib0
>   DEVICE=ib0
>   BOOTPROTO=static
>   DHCPCLASS=
>   IPADDR=192.168.200.232
>   NETMASK=255.255.255.0
>   ONBOOT=yes
> _______________________________________________
> ewg mailing list
> ewg at lists.openfabrics.org
> http://lists.openfabrics.org/cgi-bin/mailman/listinfo/ewg


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.openfabrics.org/pipermail/ewg/attachments/20101126/4f243796/attachment.html>