[openib-general] IB initialization

[Bob Woodruff]

Chris wrote, 
>As an IBGD user, I'm used to an init file in /etc/init.d... but there was
none.

>From the wiki, I was able to glean:

            Make the udev file:

># cat > /etc/udev/rules.d/40-infiniband.rules
>KERNEL="umad*", NAME="infiniband/%k"
>KERNEL="issm*", NAME="infiniband/%k"

             Install some modules:

>modprobe ib_ucm
>modprobe ib_cm
>modprobe ib_uverbs
>modprobe ib_umad


>Is there a definitive guide on the initialization of the drivers and
fabric?

FYI to anyone else trying to get things loaded and running.
Here is an init.d startup script that I use to load and start the IB
drivers.
You can use it and or edit it to load the drivers that you want. 
My script makes the dev nodes manually, but if you have udev, you can use
that instead.

#!/bin/sh
#
# ib : A script to control openib.org kernel module start
#


# Set variables
module1=ib_mthca
module2=ib_mad
module3=ib_sa
module4=ib_ipoib
module5=ib_uverbs
module6=ib_umad
module7=ib_cm 
module8=ib_ucm 
module9=ib_sdp
module10=ib_srp
module11=rdma_cm
module12=rdma_ucm
# module13=kdapl       depreciated
module14=iscsi_tcp
module15=ib_iser
device=infiniband
mode=666


# Set default module parameters
det_max_pages_percent=0
det_retry_time=0
det_window_size=0



usage()
{
  echo "Usage: $0 {start|stop|restart|reload} [module_parameters]"
}



verify_root_privilege()
{
  if [ $UID != 0 ]; then
    echo "You must be root to modify $module state"
    exit 1
  fi
}



start()
{
  verify_root_privilege

  kernel_ver=$(uname -r)
 
module1_path=/lib/modules/$kernel_ver/kernel/drivers/infiniband/hw/mthca/$mo
dule1.ko
 
module2_path=/lib/modules/$kernel_ver/kernel/drivers/infiniband/core/$module
2.ko
 
module3_path=/lib/modules/$kernel_ver/kernel/drivers/infiniband/core/$module
3.ko
 
module4_path=/lib/modules/$kernel_ver/kernel/drivers/infiniband/ulp/ipoib/$m
odule4.ko
 
module5_path=/lib/modules/$kernel_ver/kernel/drivers/infiniband/core/$module
5.ko
 
module6_path=/lib/modules/$kernel_ver/kernel/drivers/infiniband/core/$module
6.ko
 
module7_path=/lib/modules/$kernel_ver/kernel/drivers/infiniband/core/$module
7.ko
 
module8_path=/lib/modules/$kernel_ver/kernel/drivers/infiniband/core/$module
8.ko
 
module9_path=/lib/modules/$kernel_ver/kernel/drivers/infiniband/ulp/sdp/$mod
ule9.ko
 
module10_path=/lib/modules/$kernel_ver/kernel/drivers/infiniband/ulp/srp/$mo
dule10.ko
 
module11_path=/lib/modules/$kernel_ver/kernel/drivers/infiniband/core/$modul
e11.ko
 
module12_path=/lib/modules/$kernel_ver/kernel/drivers/infiniband/core/$modul
e12.ko
#
module13_path=/lib/modules/$kernel_ver/kernel/drivers/infiniband/ulp/kdapl/$
module13.ko
  module14_path=/lib/modules/$kernel_ver/kernel/drivers/scsi/$module14.ko
 
module15_path=/lib/modules/$kernel_ver/kernel/drivers/infiniband/ulp/iser/$m
odule15.ko

  if test -e $module1_path; then
    echo "Loading $module1"
    sudo /sbin/modprobe $module1 $@
  else
    echo "Module $module not found ($module1_path does not exist)!"
  fi
  if test -e $module2_path; then
    echo "Loading $module2"
    sudo /sbin/modprobe $module2  $@
  else
    echo "Module $module not found ($module2_path does not exist)!"
  fi

  if test -e $module3_path; then
    echo "Loading $module3"
    sudo /sbin/modprobe $module3  $@
  else
    echo "Module $module not found ($module3_path does not exist)!"
  fi

  if test -e $module4_path; then
    echo "Loading $module4"
    sudo /sbin/modprobe $module4  $@
  else
    echo "Module $module not found ($module4_path does not exist)!"
  fi

  if test -e $module5_path; then
    echo "Loading $module5"
    sudo /sbin/modprobe $module5  $@
  else
    echo "Module $module not found ($module5_path does not exist)!"
  fi

  if test -e $module6_path; then
    echo "Loading $module6"
    sudo /sbin/modprobe $module6  $@
  else
    echo "Module $module not found ($module6_path does not exist)!"
  fi

  if test -e $module7_path; then
    echo "Loading $module7"
    sudo /sbin/modprobe $module7  $@
  else
    echo "Module $module not found ($module7_path does not exist)!"
  fi

  if test -e $module8_path; then
    echo "Loading $module8"
    sudo /sbin/modprobe $module8  $@
  else
    echo "Module $module not found ($module8_path does not exist)!"
  fi

  if test -e $module9_path; then
    echo "Loading $module9"
    sudo /sbin/modprobe $module9  $@
  else
    echo "Module $module not found ($module9_path does not exist)!"
  fi
  if test -e $module10_path; then
    echo "Loading $module10"
    sudo /sbin/modprobe $module10  $@
  else
    echo "Module $module not found ($module10_path does not exist)!"
  fi
  if test -e $module11_path; then
    echo "Loading $module11"
    sudo /sbin/modprobe $module11  $@
  else
    echo "Module $module not found ($module11_path does not exist)!"
  fi
  if test -e $module12_path; then
    echo "Loading $module12"
    sudo /sbin/modprobe $module12  $@
  else
    echo "Module $module not found ($module12_path does not exist)!"
  fi
#  if test -e $module13_path; then
#    echo "Loading $module13"
#    sudo /sbin/modprobe $module13  $@
#  else
#    echo "Module $module not found ($module13_path does not exist)!"
#  fi
  if test -e $module14_path; then
    echo "Loading $module14"
    sudo /sbin/modprobe $module14  $@
  else
   echo "Module $module not found ($module14_path does not exist)!"
  fi
  if test -e $module15_path; then
    echo "Loading $module15"
    sudo /sbin/modprobe $module15  $@
  else
    echo "Module $module not found ($module15_path does not exist)!"
  fi
#endif

# remove stale device
  rm -rf /dev/$device

mkdir /dev/infiniband
/bin/mknod /dev/infiniband/umad0 c 231 0
/bin/mknod /dev/infiniband/uverbs0 c 231 192
/bin/mknod /dev/infiniband/uverbs1 c 231 193
/bin/mknod /dev/infiniband/ucm0 c 231 224
/bin/mknod /dev/infiniband/uat c 231 191
/bin/mknod /dev/infiniband/rdma_cm c 10 62
chmod 777 /dev/infiniband/*

/usr/local/bin/ifup-ib ib0
/usr/local/bin/ifup-ib ib1

  return 0
}



stop()
{
  verify_root_privilege

  echo "Removing $module15"
  /sbin/rmmod $module15.ko
  echo "Removing $module14"
  /sbin/rmmod $module14.ko
#  echo "Removing $module13"
#  /sbin/rmmod $module13.ko
  echo "Removing $module12"
  /sbin/rmmod $module12.ko
  echo "Removing $module11"
  /sbin/rmmod $module11.ko
  echo "Removing $module10"
  /sbin/rmmod $module10.ko
  echo "Removing $module9"
  /sbin/rmmod $module9.ko
  echo "Removing $module8"
  /sbin/rmmod $module8.ko
  echo "Removing $module7"
  /sbin/rmmod $module7.ko
  echo "Removing $module6"
  /sbin/rmmod $module6.ko
  echo "Removing $module5"
  /sbin/rmmod $module5.ko
  echo "Removing $module4"
  /sbin/rmmod $module4.ko
  echo "Removing $module3"
  /sbin/rmmod ib_local_sa
  /sbin/rmmod $module3.ko
  echo "Removing $module1"
  /sbin/rmmod $module1.ko
  echo "Removing $module2"
  /sbin/rmmod $module2.ko

  rm -rf /dev/$device

  return 0
}



restart()
{
  stop
  start $@
}



# "main"
next_state=$1
shift

# See how we were called
case "$next_state" in
  start)
    start $@
    ;;
  stop)
    stop
    ;;
  restart|reload)
    restart $@
    ;;
  *)
    usage
    exit 1
esac