[ofa-general] Re: [PATCH] opensm & osm_console: modified console framework to support multiple connections

Sasha Khapyorsky sashak at voltaire.com
Sat Oct 27 18:02:26 PDT 2007


Hi Tim,

Sorry about very long delay with reviewing this.

On 16:52 Mon 15 Oct     , Timothy A. Meier wrote:
>   This patch is setting up for adding Remote/Secure Console capability using 
>  SSL/TSL (we need at LLNL).

Thanks for doing this - it is great thing to secure OpenSM console.

>   Its a big patch because I changed to an abstract server model, instead of 
>  the original
>  single connection and synchronous model.  There is no significant functional 
>  difference (yet).

It is hard to understand how such abstraction model serves us without
seeing the rest of SSL/TSL code. Probably it is better idea to issue
whole patch series? Anyway some initial comments are below.

>  ========
>  From cb69c1e2c8ea526bcb1e81d079bfa787eda09ba8 Mon Sep 17 00:00:00 2001
>  From: Tim Meier <meier3 at llnl.gov>
>  Date: Mon, 15 Oct 2007 16:08:10 -0700
>  Subject: [PATCH] opensm & osm_console: modified console framework to support 
>  multiple connections
> 
>  Provided an abstract console service that supports the current connection 
>  types
>  (local, loopback, socket) as well as supporting the addition of a secure
>  connection type.
> 
>  * A server implementation supports multiple connections, and reduces the
>  posibility of an inadvertant denial of service (currently vulnerable).
>
>  * An IO abstraction (CIO) is employed to facilitate the future 
>  implementation
>  of a secure socket (SSL / TSL) connection, while maintaining backward
>  compatibility.

Would be nice to not mix two things in one patch - "one patch per
thought" makes it easier to review and submit.

> 
>  Signed-off-by: Tim Meier <meier3 at llnl.gov>
>  ---
>  opensm/include/opensm/osm_console.h |   35 +-
>  opensm/opensm/main.c                |   77 ++-
>  opensm/opensm/osm_console.c         | 1500 
>  +++++++++++++++++++++++++----------
>  3 files changed, 1177 insertions(+), 435 deletions(-)
> 
>  diff --git a/opensm/include/opensm/osm_console.h 
>  b/opensm/include/opensm/osm_console.h
>  index 33e41e7..75111a4 100644
>  --- a/opensm/include/opensm/osm_console.h
>  +++ b/opensm/include/opensm/osm_console.h
>  @@ -49,6 +49,14 @@
>  #define OSM_DEFAULT_CONSOLE      OSM_DISABLE_CONSOLE
>  #define OSM_DEFAULT_CONSOLE_PORT 10000
>  #define OSM_DAEMON_NAME          "opensm"
>  +#define OSM_QUIT_CMD             "quit"
>  +#define OSM_LOOP_PERIOD_SEC      2
>  +
>  +#define CIO_BUFSIZE          1024
>  +#define CIO_INFO_SIZE         128
>  +#define CIO_NOTE_SIZE          64
>  +#define CIO_MAX_CONNECTS        5
>  +#define CIO_CONNECTION_PORT 10000
>  #ifdef __cplusplus
>  #  define BEGIN_C_DECLS extern "C" {
>  @@ -59,10 +67,29 @@
>  #endif                /* __cplusplus */
>  BEGIN_C_DECLS
>  -void osm_console_init(osm_subn_opt_t * opt, osm_opensm_t * p_osm);
>  -void osm_console(osm_opensm_t * p_osm);
>  -void osm_console_prompt(FILE * out);
>  -void osm_console_close_socket(osm_opensm_t * p_osm);
>  +
>  +/* TODO move when fully implemented */
>  +typedef struct _CIO_t
>  +{
>  +  int  fd;  // file descriptor (socket)
>  +  FILE *out;
>  +  FILE *err;
>  +  FILE *in;
>  +  struct pollfd *pfd;
>  +} CIO_t;
>  +
>  +int osm_console_server(osm_subn_opt_t *p_opt, osm_opensm_t *p_osm);
>  +void osm_console_server_init(osm_subn_opt_t *opt, osm_opensm_t *p_osm);
>  +void osm_console_server_destroy(osm_opensm_t *p_osm);
>  +int is_console_enabled(osm_subn_opt_t *p_opt);
>  +
>  +/* TODO move along with other IO abstraction code */
>  +int cio_printf( CIO_t *cio, const char *format, ...);
>  +int cio_flush( CIO_t *cio);
>  +int cio_getline( char **lineptr, size_t *n, CIO_t *cio);
>  +int cio_open( CIO_t *cio);
>  +int cio_close( CIO_t *cio);
>  +int cio_poll(CIO_t *cio, int timeout);

Later I see that all cio_* and CIO_* stuff is used only in
osm_console.c, then I think this all should be moved to this file,
local function should be static, etc..

Another thing, please try to not break existing coding style (it is
described in opensm/doc/opensm-coding-style.txt), in many cases you can
use opensm/opensm/osm_indent script to format the code.

>  END_C_DECLS
>  #endif                /* _OSM_CONSOLE_H_ */
>  diff --git a/opensm/opensm/main.c b/opensm/opensm/main.c
>  index 0250551..b744157 100644
>  --- a/opensm/opensm/main.c
>  +++ b/opensm/opensm/main.c
>  @@ -229,11 +229,13 @@ void show_usage(void)
>             "          SMPs.\n"
>             "          Without -maxsmps, OpenSM defaults to a maximum of\n"
>             "          4 outstanding SMPs.\n\n");
>  -    printf("-console [off|local"
>  #ifdef ENABLE_OSM_CONSOLE_SOCKET
>  -           "|socket|loopback"
>  +  printf("-console [%s|%s|%s|%s]", OSM_DISABLE_CONSOLE, OSM_LOCAL_CONSOLE,
>  +         OSM_REMOTE_CONSOLE, OSM_LOOPBACK_CONSOLE);
>  +#else
>  +  printf("-console [%s|%s]", OSM_DISABLE_CONSOLE, OSM_LOCAL_CONSOLE);
>  #endif
>  -           "]\n          This option activates the OpenSM console (default 
>  off).\n\n");
>  +    printf("]\n          This option activates the OpenSM console (default 
>  off).\n\n");
>  #ifdef ENABLE_OSM_CONSOLE_SOCKET
>      printf("-console-port <port>\n"
>             "          Specify an alternate telnet port for the console 
>  (default %d).\n\n",
>  @@ -566,6 +568,45 @@ static int daemonize(osm_opensm_t * osm)
>      return 0;
>  }
>  +/* simple server to provide an interface to support
>  + *  interactive (and non-interactive) commands + *   loop here until an 
>  exit signal is received
>  + *
>  + * currently just support a command console
>  + */
>  +void osm_opensm_server(osm_subn_opt_t *p_opt, osm_opensm_t *p_osm)
>  +{
>  +  if(is_console_enabled(p_opt))
>  +    osm_console_server_init(p_opt, p_osm);
>  +
>  +  /*
>  +   Sit here forever - dwelling or running the server
>  +   */
>  +  while (!osm_exit_flag)
>  +  {
>  +    if(is_console_enabled(p_opt))
>  +      osm_console_server(p_opt, p_osm);
>  +    else
>  +      cl_thread_suspend( 10000);
>  +
>  +    if (osm_usr1_flag)
>  +    {
>  +      osm_usr1_flag = 0;
>  +      osm_log_reopen_file(&(p_osm->log));
>  +    }
>  +    if (osm_hup_flag)
>  +    {
>  +      osm_hup_flag = 0;
>  +      /* a HUP signal should only start a new heavy sweep */
>  +      p_osm->subn.force_immediate_heavy_sweep = TRUE;
>  +      osm_opensm_sweep(p_osm);
>  +    }
>  +  }
>  + +  if(is_console_enabled(p_opt))
>  +    osm_console_server_destroy(p_osm);
>  +}
>  +
>  /**********************************************************************
>   **********************************************************************/
>  int main(int argc, char *argv[])
>  @@ -1034,34 +1075,8 @@ int main(int argc, char *argv[])
>                  osm_exit_flag = 1;
>          }
>      } else {
>  -        osm_console_init(&opt, &osm);
>  -
>  -        /*
>  -           Sit here forever
>  -         */
>  -        while (!osm_exit_flag) {
>  -            if (strcmp(opt.console, OSM_LOCAL_CONSOLE) == 0
>  -#ifdef ENABLE_OSM_CONSOLE_SOCKET
>  -                || strcmp(opt.console, OSM_REMOTE_CONSOLE) == 0
>  -                || strcmp(opt.console, OSM_LOOPBACK_CONSOLE) == 0
>  -#endif
>  -                )
>  -                osm_console(&osm);
>  -            else
>  -                cl_thread_suspend(10000);
>  -
>  -            if (osm_usr1_flag) {
>  -                osm_usr1_flag = 0;
>  -                osm_log_reopen_file(&osm.log);
>  -            }
>  -            if (osm_hup_flag) {
>  -                osm_hup_flag = 0;
>  -                /* a HUP signal should only start a new heavy sweep */
>  -                osm.subn.force_immediate_heavy_sweep = TRUE;
>  -                osm_opensm_sweep(&osm);
>  -            }
>  -        }
>  -        osm_console_close_socket(&osm);
>  +    // start a server that runs indefinately
>  +    osm_opensm_server(&opt, &osm);
>      }
>  #if 0
>  diff --git a/opensm/opensm/osm_console.c b/opensm/opensm/osm_console.c
>  index c6e02ab..9d62774 100644
>  --- a/opensm/opensm/osm_console.c
>  +++ b/opensm/opensm/osm_console.c
>  @@ -38,15 +38,16 @@
>  #define _GNU_SOURCE        /* for getline */
>  #include <stdio.h>
>  #include <stdlib.h>
>  +#include <stdarg.h>
>  #include <sys/poll.h>
>  #include <sys/types.h>
>  #include <sys/socket.h>
>  #include <netdb.h>
>  #ifdef ENABLE_OSM_CONSOLE_SOCKET
>  #include <tcpd.h>
>  -#endif
>  #include <arpa/inet.h>
>  #include <netinet/in.h>
>  +#endif
>  #include <unistd.h>
>  #include <errno.h>
>  #include <ctype.h>
>  @@ -57,20 +58,113 @@
>  #include <complib/cl_passivelock.h>
>  #include <opensm/osm_perfmgr.h>
>  +typedef struct _LoopCmd
>  +{
>  +  int    on;
>  +  int    running;
>  +  int    delay_s;
>  +  void (*loop_function)(osm_opensm_t *p_osm, CIO_t *out);
>  +  cl_thread_t  loopThread; // a specific thread for each looping cmd
>  +} LoopCmd;
>  +
>  +// unique attributes for each connection
>  +typedef struct _osm_console_thread_t
>  +{
>  +  int used;
>  +  unsigned short int port;
>  +  int authorized;
>  +  int state;
>  +  char name[CIO_INFO_SIZE];
>  +  char in_buff[CIO_BUFSIZE];
>  +  char out_buff[CIO_BUFSIZE];
>  +  char client_type[CIO_NOTE_SIZE];  // maps to option->console 
>  (off|local|socket)
>  +  char client_ip[CIO_NOTE_SIZE];
>  +  char client_hn[CIO_INFO_SIZE];
>  +  unsigned int thread_num;  // a unique ever increasing number +  
>  osm_opensm_t *p_osm;     // the global opensm singleton (protect with lock)
>  +  CIO_t io;                // the io streams for the connection
>  +  LoopCmd loop_command;
>  +  cl_thread_t  consoleThread; // a specific thread each console connection
>  +  struct timeval connect_time;
>  +} osm_console_thread_t;

I think this introduces CIO_MAX_CONNECTS new threads + for loop commands.
What about to do all in one thread - to use select() or poll() with
timeout on multiple file descriptors? This will "reserve" another CPUs
for running another OpenSM things. Another potential problem is multi
thread synchronizations - we had (and still have) a lot of issues in this
area.

>  +
>  struct command {
>  -    char *name;
>  -    void (*help_function) (FILE * out, int detail);
>  -    void (*parse_function) (char **p_last, osm_opensm_t * p_osm,
>  -                FILE * out);
>  +  char *name;
>  +  void (*help_function)(CIO_t *out, int detail);
>  +  void (*parse_function)(char **p_last, osm_console_thread_t *p_oct, CIO_t 
>  *out);
>  };
>  -struct {
>  -    int on;
>  -    int delay_s;
>  -    time_t previous;
>  -    void (*loop_function) (osm_opensm_t * p_osm, FILE * out);
>  -} loop_command = {
>  -on: 0, delay_s: 2, loop_function:NULL};
>  +/* connection pool for remote clients - currently only consoles */
>  +static osm_console_thread_t ConsoleThreadPool[CIO_MAX_CONNECTS];
>  +static cl_plock_t ThreadLock;
>  +static volatile unsigned int cio_thread_counter = 0;
>  +static struct timeval ServerTime;

Would be nice to avoid using non-constant static/global variables.
Instead we could keep needed per OpenSM session info in allocated
structure.

>  +
>  +/**********************************************************************
>  + * convenience function
>  + **********************************************************************/
>  +CIO_t* getCIO(osm_console_thread_t *oct)

This function should be static?

>  +{
>  +  return &oct->io;
>  +}
>  +
>  +/**********************************************************************
>  + * thread pool primitive: counts the number currently in use
>  + **********************************************************************/
>  +int num_console_threads(void)

Ditto (and many others below)

>  +{
>  +  // count them up
>  +
>  +  int i;
>  +  int num = 0;
>  + +  cl_plock_acquire(&ThreadLock);
>  +  for(i = 0; i <  CIO_MAX_CONNECTS; ++i)
>  +  {
>  +    if(ConsoleThreadPool[i].used != 0)
>  +      num++;
>  +  }
>  +  cl_plock_release(&ThreadLock);
>  + +  return num;
>  +}
>  +
>  +/**********************************************************************
>  + * thread pool primitive: the current value reflects the number of
>  + * connection attempts made since program execution.
>  + **********************************************************************/
>  +unsigned int get_console_thread_counter(void)
>  +{
>  +  return cio_thread_counter;
>  +}
>  +
>  +int is_loopback(char* str)
>  +{
>  +  // convenience - checks if socket based connection
>  + if(str)
>  +   return (strcmp(str, OSM_LOOPBACK_CONSOLE) == 0);
>  +return 0;
>  +}
>  +
>  +int is_remote(char* str)
>  +{
>  +  // convenience - checks if socket based connection
>  + if(str)
>  +   return (strcmp(str, OSM_REMOTE_CONSOLE) == 0)
>  +       || is_loopback(str);
>  +return 0;
>  +}
>  +
>  +int is_console_enabled(osm_subn_opt_t *p_opt)
>  +{
>  +  // checks for a variety of types of consoles - default is off or 0
>  + if(p_opt)
>  +   return ((strcmp(p_opt->console, OSM_LOCAL_CONSOLE) == 0)
>  +       || (strcmp(p_opt->console, OSM_LOOPBACK_CONSOLE) == 0)
>  +       || (strcmp(p_opt->console, OSM_REMOTE_CONSOLE) == 0));
>  +return 0;
>  +}
>  +
>  static const struct command console_cmds[];
>  @@ -79,114 +173,103 @@ static inline char *next_token(char **p_last)
>      return strtok_r(NULL, " \t\n\r", p_last);
>  }
>  -static void help_command(FILE * out, int detail)
>  +static void help_command(CIO_t *out, int detail)
>  {
>      int i;
>  -    fprintf(out, "Supported commands and syntax:\n");
>  -    fprintf(out, "help [<command>]\n");
>  +    cio_printf(out, "Supported commands and syntax:\n");
>  +    cio_printf(out, "help [<command>]\n");
>      /* skip help command */
>      for (i = 1; console_cmds[i].name; i++)
>          console_cmds[i].help_function(out, 0);
>  }
>  -static void help_quit(FILE * out, int detail)
>  +static void help_quit(CIO_t *out, int detail)
>  {
>  -    fprintf(out, "quit (not valid in local mode; use ctl-c)\n");
>  +    cio_printf(out, "%s -- stops the console\n", OSM_QUIT_CMD);
>  +  if (detail) {
>  +    cio_printf(out, "  OpenSM will continue, to kill; \n");
>  +    cio_printf(out, "    use ctrl-C in local mode or\n");
>  +    cio_printf(out, "    kill the process\n");
>  +  }
>  }
>  -static void help_loglevel(FILE * out, int detail)
>  +
>  +static void help_loglevel(CIO_t *out, int detail)
>  {
>  -    fprintf(out, "loglevel [<log-level>]\n");
>  +    cio_printf(out, "loglevel [<log-level>]\n");
>      if (detail) {
>  -        fprintf(out, "   log-level is OR'ed from the following\n");
>  -        fprintf(out, "   OSM_LOG_NONE             0x%02X\n",
>  -            OSM_LOG_NONE);
>  -        fprintf(out, "   OSM_LOG_ERROR            0x%02X\n",
>  -            OSM_LOG_ERROR);
>  -        fprintf(out, "   OSM_LOG_INFO             0x%02X\n",
>  -            OSM_LOG_INFO);
>  -        fprintf(out, "   OSM_LOG_VERBOSE          0x%02X\n",
>  -            OSM_LOG_VERBOSE);
>  -        fprintf(out, "   OSM_LOG_DEBUG            0x%02X\n",
>  -            OSM_LOG_DEBUG);
>  -        fprintf(out, "   OSM_LOG_FUNCS            0x%02X\n",
>  -            OSM_LOG_FUNCS);
>  -        fprintf(out, "   OSM_LOG_FRAMES           0x%02X\n",
>  -            OSM_LOG_FRAMES);
>  -        fprintf(out, "   OSM_LOG_ROUTING          0x%02X\n",
>  -            OSM_LOG_ROUTING);
>  -        fprintf(out, "   OSM_LOG_SYS              0x%02X\n",
>  -            OSM_LOG_SYS);
>  -        fprintf(out, "\n");
>  -        fprintf(out, "   OSM_LOG_DEFAULT_LEVEL    0x%02X\n",
>  -            OSM_LOG_DEFAULT_LEVEL);
>  +        cio_printf(out, "   log-level is OR'ed from the following\n");
>  +        cio_printf(out, "   OSM_LOG_NONE             0x%02X\n", 
>  OSM_LOG_NONE);
>  +        cio_printf(out, "   OSM_LOG_ERROR            0x%02X\n", 
>  OSM_LOG_ERROR);
>  +        cio_printf(out, "   OSM_LOG_INFO             0x%02X\n", 
>  OSM_LOG_INFO);
>  +        cio_printf(out, "   OSM_LOG_VERBOSE          0x%02X\n", 
>  OSM_LOG_VERBOSE);
>  +        cio_printf(out, "   OSM_LOG_DEBUG            0x%02X\n", 
>  OSM_LOG_DEBUG);
>  +        cio_printf(out, "   OSM_LOG_FUNCS            0x%02X\n", 
>  OSM_LOG_FUNCS);
>  +        cio_printf(out, "   OSM_LOG_FRAMES           0x%02X\n", 
>  OSM_LOG_FRAMES);
>  +        cio_printf(out, "   OSM_LOG_ROUTING          0x%02X\n", 
>  OSM_LOG_ROUTING);
>  +        cio_printf(out, "   OSM_LOG_SYS              0x%02X\n", 
>  OSM_LOG_SYS);
>  +        cio_printf(out, "\n");
>  +        cio_printf(out, "   OSM_LOG_DEFAULT_LEVEL    0x%02X\n", 
>  OSM_LOG_DEFAULT_LEVEL);
>      }
>  }
>  -static void help_priority(FILE * out, int detail)
>  +static void help_priority(CIO_t *out, int detail)
>  {
>  -    fprintf(out, "priority [<sm-priority>]\n");
>  +    cio_printf(out, "priority [<sm-priority>]\n");
>  }
>  -static void help_resweep(FILE * out, int detail)
>  +static void help_resweep(CIO_t *out, int detail)
>  {
>  -    fprintf(out, "resweep [heavy|light]\n");
>  +    cio_printf(out, "resweep [heavy|light]\n");
>  }
>  -static void help_status(FILE * out, int detail)
>  +static void help_status(CIO_t *out, int detail)
>  {
>  -    fprintf(out, "status [loop]\n");
>  +    cio_printf(out, "status [loop]\n");
>      if (detail) {
>  -        fprintf(out, "   loop -- type \"q<ret>\" to quit\n");
>  +        cio_printf(out, "   loop -- type \"q<ret>\" to quit\n");
>      }
>  }
>  -static void help_logflush(FILE * out, int detail)
>  +static void help_logflush(CIO_t *out, int detail)
>  {
>  -    fprintf(out, "logflush -- flush the opensm.log file\n");
>  +    cio_printf(out, "logflush -- flush the opensm.log file\n");
>  }
>  -static void help_querylid(FILE * out, int detail)
>  +static void help_querylid(CIO_t *out, int detail)
>  {
>  -    fprintf(out,
>  -        "querylid lid -- print internal information about the lid 
>  specified\n");
>  +    cio_printf(out,
>  +    "querylid lid -- print internal information about the lid 
>  specified\n");
>  }
>  -static void help_portstatus(FILE * out, int detail)
>  +static void help_portstatus(CIO_t *out, int detail)
>  {
>  -    fprintf(out, "portstatus [ca|switch|router]\n");
>  +    cio_printf(out, "portstatus [ca|switch|router]\n");
>      if (detail) {
>  -        fprintf(out, "summarize port status\n");
>  -        fprintf(out,
>  -            "   [ca|switch|router] -- limit the results to the node type 
>  specified\n");
>  +        cio_printf(out, "summarize port status\n");
>  +        cio_printf(out, "   [ca|switch|router] -- limit the results to the 
>  node type specified\n");
>      }
>  }
>  #ifdef ENABLE_OSM_PERF_MGR
>  -static void help_perfmgr(FILE * out, int detail)
>  +static void help_perfmgr(CIO_t *out, int detail)
>  {
>  -    fprintf(out,
>  -        "perfmgr 
>  [enable|disable|clear_counters|dump_counters|sweep_time[seconds]]\n");
>  +    cio_printf(out, "perfmgr 
>  [enable|disable|clear_counters|dump_counters|sweep_time[seconds]]\n");
>      if (detail) {
>  -        fprintf(out,
>  -            "perfmgr -- print the performance manager state\n");
>  -        fprintf(out,
>  -            "   [enable|disable] -- change the perfmgr state\n");
>  -        fprintf(out,
>  -            "   [sweep_time] -- change the perfmgr sweep time (requires 
>  [seconds] option)\n");
>  -        fprintf(out,
>  -            "   [clear_counters] -- clear the counters stored\n");
>  -        fprintf(out,
>  -            "   [dump_counters [mach]] -- dump the counters (optionally in 
>  [mach]ine readable format)\n");
>  +        cio_printf(out, "perfmgr -- print the performance manager 
>  state\n");
>  +        cio_printf(out, "   [enable|disable] -- change the perfmgr 
>  state\n");
>  +        cio_printf(out, "   [sweep_time] -- change the perfmgr sweep time 
>  (requires [seconds] option)\n");
>  +        cio_printf(out, "   [clear_counters] -- clear the counters 
>  stored\n");
>  +        cio_printf(out, "   [dump_counters [mach]] -- dump the counters 
>  (optionally in [mach]ine readable format)\n");
>      }
>  }
>  #endif                /* ENABLE_OSM_PERF_MGR */
>  /* more help routines go here */
>  -static void help_parse(char **p_last, osm_opensm_t * p_osm, FILE * out)
>  +static void help_parse(char **p_last, osm_console_thread_t *p_oct, CIO_t 
>  *out)
>  {
>      char *p_cmd;
>      int i, found = 0;
>  @@ -203,21 +286,21 @@ static void help_parse(char **p_last, osm_opensm_t * 
>  p_osm, FILE * out)
>              }
>          }
>          if (!found) {
>  -            fprintf(out, "%s : Command not found\n\n", p_cmd);
>  +            cio_printf(out, "%s : Command not found\n\n", p_cmd);
>              help_command(out, 0);
>          }
>      }
>  }
>  -static void loglevel_parse(char **p_last, osm_opensm_t * p_osm, FILE * out)
>  +static void loglevel_parse(char **p_last, osm_console_thread_t *p_oct, 
>  CIO_t *out)
>  {
>  +  osm_opensm_t *p_osm = p_oct->p_osm;
>      char *p_cmd;
>      int level;
>      p_cmd = next_token(p_last);
>      if (!p_cmd)
>  -        fprintf(out, "Current log level is 0x%x\n",
>  -            osm_log_get_level(&p_osm->log));
>  +        cio_printf(out, "Current log level is 0x%x\n", 
>  osm_log_get_level(&p_osm->log));

At least here your mailer wraps the line :(

>      else {
>          /* Handle x, 0x, and decimal specification of log level */
>          if (!strncmp(p_cmd, "x", 1)) {
>  @@ -231,31 +314,29 @@ static void loglevel_parse(char **p_last, osm_opensm_t 
>  * p_osm, FILE * out)
>                  level = strtol(p_cmd, NULL, 10);
>          }
>          if ((level >= 0) && (level < 256)) {
>  -            fprintf(out, "Setting log level to 0x%x\n", level);
>  +            cio_printf(out, "Setting log level to 0x%x\n", level);
>              osm_log_set_level(&p_osm->log, level);
>          } else
>  -            fprintf(out, "Invalid log level 0x%x\n", level);
>  +            cio_printf(out, "Invalid log level 0x%x\n", level);
>      }
>  }
>  -static void priority_parse(char **p_last, osm_opensm_t * p_osm, FILE * out)
>  +static void priority_parse(char **p_last, osm_console_thread_t *p_oct, 
>  CIO_t *out)
>  {
>  +  osm_opensm_t *p_osm = p_oct->p_osm;
>      char *p_cmd;
>      int priority;
>      p_cmd = next_token(p_last);
>      if (!p_cmd)
>  -        fprintf(out, "Current sm-priority is %d\n",
>  -            p_osm->subn.opt.sm_priority);
>  +        cio_printf(out, "Current sm-priority is %d\n", 
>  p_osm->subn.opt.sm_priority);
>      else {
>          priority = strtol(p_cmd, NULL, 0);
>          if (0 > priority || 15 < priority)
>  -            fprintf(out,
>  -                "Invalid sm-priority %d; must be between 0 and 15\n",
>  -                priority);
>  +            cio_printf(out, "Invalid sm-priority %d; must be between 0 and 
>  15\n", priority);
>          else {
>  -            fprintf(out, "Setting sm-priority to %d\n", priority);
>  -            p_osm->subn.opt.sm_priority = (uint8_t) priority;
>  +            cio_printf(out, "Setting sm-priority to %d\n", priority);
>  +            p_osm->subn.opt.sm_priority = (uint8_t)priority;
>              /* Does the SM state machine need a kick now ? */
>          }
>      }
>  @@ -371,24 +452,23 @@ static char *sm_state_mgr_str(osm_sm_state_t state)
>      }
>  }
>  -static void print_status(osm_opensm_t * p_osm, FILE * out)
>  +static void print_status(osm_opensm_t *p_osm, CIO_t *out)
>  {
>      if (out) {
>  -        fprintf(out, "   OpenSM Version     : %s\n", OSM_VERSION);
>  -        fprintf(out, "   SM State/Mgr State : %s/%s\n",
>  +        cio_printf(out, "   OpenSM Version     : %s\n", OSM_VERSION);
>  +        cio_printf(out, "   SM State/Mgr State : %s/%s\n",
>              sm_state_str(p_osm->subn.sm_state),
>              sm_state_mgr_str(p_osm->sm.state_mgr.state));
>  -        fprintf(out, "   SA State           : %s\n",
>  +        cio_printf(out, "   SA State           : %s\n",
>              sa_state_str(p_osm->sa.state));
>  -        fprintf(out, "   Routing Engine     : %s\n",
>  -            p_osm->routing_engine.name ? p_osm->routing_engine.
>  -            name : "null (min-hop)");
>  +        cio_printf(out, "   Routing Engine     : %s\n",
>  +            p_osm->routing_engine.name ? p_osm->routing_engine.name : "null 
>  (min-hop)");
>  #ifdef ENABLE_OSM_PERF_MGR
>  -        fprintf(out, "\n   PerfMgr state/sweep state : %s/%s\n",
>  +        cio_printf(out, "\n   PerfMgr state/sweep state : %s/%s\n",
>              osm_perfmgr_get_state_str(&(p_osm->perfmgr)),
>              osm_perfmgr_get_sweep_state_str(&(p_osm->perfmgr)));
>  #endif
>  -        fprintf(out, "\n   MAD stats\n"
>  +        cio_printf(out, "\n   MAD stats\n"
>              "   ---------\n"
>              "   QP0 MADs outstanding           : %d\n"
>              "   QP0 MADs outstanding (on wire) : %d\n"
>  @@ -412,7 +492,7 @@ static void print_status(osm_opensm_t * p_osm, FILE * 
>  out)
>              p_osm->stats.sa_mads_sent,
>              p_osm->stats.sa_mads_rcvd_unknown,
>              p_osm->stats.sa_mads_ignored);
>  -        fprintf(out, "\n   Subnet flags\n"
>  +        cio_printf(out, "\n   Subnet flags\n"
>              "   ------------\n"
>              "   Ignore existing lfts           : %d\n"
>              "   Subnet Init errors             : %d\n"
>  @@ -426,32 +506,24 @@ static void print_status(osm_opensm_t * p_osm, FILE * 
>  out)
>              p_osm->subn.moved_to_master_state,
>              p_osm->subn.first_time_master_sweep,
>              p_osm->subn.coming_out_of_standby);
>  -        fprintf(out, "\n");
>  -    }
>  -}
>  -
>  -static int loop_command_check_time(void)
>  -{
>  -    time_t cur = time(NULL);
>  -    if ((loop_command.previous + loop_command.delay_s) < cur) {
>  -        loop_command.previous = cur;
>  -        return (1);
>  +        cio_printf(out, "\n");
>      }
>  -    return (0);
>  }
>  -static void status_parse(char **p_last, osm_opensm_t * p_osm, FILE * out)
>  +static void status_parse(char **p_last, osm_console_thread_t *p_oct, CIO_t 
>  *out)
>  {
>  +  osm_opensm_t *p_osm = p_oct->p_osm;
>      char *p_cmd;
>      p_cmd = next_token(p_last);
>      if (p_cmd) {
>          if (strcmp(p_cmd, "loop") == 0) {
>  -            fprintf(out, "Looping on status command...\n");
>  -            fflush(out);
>  -            loop_command.on = 1;
>  -            loop_command.previous = time(NULL);
>  -            loop_command.loop_function = print_status;
>  +            cio_printf(out, "Looping on status command...\n");
>  +            cio_flush(out);
>  +      p_oct->loop_command.on = 1;
>  +      p_oct->loop_command.delay_s = OSM_LOOP_PERIOD_SEC;
>  +      p_oct->loop_command.running = 0;
>  +      p_oct->loop_command.loop_function = print_status;
>          } else {
>              help_status(out, 1);
>              return;
>  @@ -460,14 +532,15 @@ static void status_parse(char **p_last, osm_opensm_t * 
>  p_osm, FILE * out)
>      print_status(p_osm, out);
>  }
>  -static void resweep_parse(char **p_last, osm_opensm_t * p_osm, FILE * out)
>  +static void resweep_parse(char **p_last, osm_console_thread_t *p_oct, CIO_t 
>  *out)
>  {
>  +  osm_opensm_t *p_osm = p_oct->p_osm;
>      char *p_cmd;
>      p_cmd = next_token(p_last);
>      if (!p_cmd ||
>          (strcmp(p_cmd, "heavy") != 0 && strcmp(p_cmd, "light") != 0)) {
>  -        fprintf(out, "Invalid resweep command\n");
>  +        cio_printf(out, "Invalid resweep command\n");
>          help_resweep(out, 1);
>      } else {
>          if (strcmp(p_cmd, "heavy") == 0) {
>  @@ -477,20 +550,21 @@ static void resweep_parse(char **p_last, osm_opensm_t 
>  * p_osm, FILE * out)
>      }
>  }
>  -static void logflush_parse(char **p_last, osm_opensm_t * p_osm, FILE * out)
>  +static void logflush_parse(char **p_last, osm_console_thread_t *p_oct, 
>  CIO_t *out)
>  {
>  -    fflush(p_osm->log.out_port);
>  +    fflush(p_oct->p_osm->log.out_port);
>  }
>  -static void querylid_parse(char **p_last, osm_opensm_t * p_osm, FILE * out)
>  +static void querylid_parse(char **p_last, osm_console_thread_t *p_oct, 
>  CIO_t *out)
>  {
>  -    int p = 0;
>  -    uint16_t lid = 0;
>  +  osm_opensm_t *p_osm = p_oct->p_osm;
>  +    int         p = 0;
>  +    uint16_t    lid = 0;
>      osm_port_t *p_port = NULL;
>      char *p_cmd = next_token(p_last);
>      if (!p_cmd) {
>  -        fprintf(out, "no LID specified\n");
>  +        cio_printf(out, "no LID specified\n");
>          help_querylid(out, 1);
>          return;
>      }
>  @@ -503,8 +577,8 @@ static void querylid_parse(char **p_last, osm_opensm_t * 
>  p_osm, FILE * out)
>      if (!p_port)
>          goto invalid_lid;
>  -    fprintf(out, "Query results for LID %d\n", lid);
>  -    fprintf(out,
>  +    cio_printf(out, "Query results for LID %d\n", lid);
>  +    cio_printf(out,
>          "   GUID                : 0x%016" PRIx64 "\n"
>          "   Node Desc           : %s\n"
>          "   Node Type           : %s\n"
>  @@ -518,20 +592,19 @@ static void querylid_parse(char **p_last, osm_opensm_t 
>  * p_osm, FILE * out)
>          p = 0;
>      else
>          p = 1;
>  -    for ( /* see above */ ; p < p_port->p_node->physp_tbl_size; p++) {
>  -        fprintf(out,
>  +    for (/* see above */; p < p_port->p_node->physp_tbl_size; p++) {
>  +        cio_printf(out,
>              "   Port %d health       : %s\n",
>              p,
>  -            p_port->p_node->physp_table[p].
>  -            healthy ? "OK" : "ERROR");
>  +            p_port->p_node->physp_table[p].healthy ? "OK" : "ERROR");
>      }
>      cl_plock_release(&p_osm->lock);
>      return;
>  -      invalid_lid:
>  +invalid_lid:
>      cl_plock_release(&p_osm->lock);
>  -    fprintf(out, "Invalid lid %d\n", lid);
>  +    cio_printf(out, "Invalid lid %d\n", lid);
>      return;
>  }
>  @@ -564,11 +637,11 @@ __tag_port_report(port_report_t ** head, uint64_t 
>  node_guid,
>          *head = rep;
>  }
>  -static void __print_port_report(FILE * out, port_report_t * head)
>  +static void __print_port_report(CIO_t *out, port_report_t *head)
>  {
>      port_report_t *item = head;
>      while (item != NULL) {
>  -        fprintf(out, "      0x%016" PRIx64 " %d (%s)\n",
>  +        cio_printf(out, "      0x%016"PRIx64" %d (%s)\n",
>              item->node_guid, item->port_num, item->print_desc);
>          port_report_t *next = item->next;
>          free(item);
>  @@ -689,10 +762,11 @@ static void __get_stats(cl_map_item_t * const 
>  p_map_item, void *context)
>      }
>  }
>  -static void portstatus_parse(char **p_last, osm_opensm_t * p_osm, FILE * 
>  out)
>  +static void portstatus_parse(char **p_last, osm_console_thread_t *p_oct, 
>  CIO_t *out)
>  {
>  -    fabric_stats_t fs;
>  -    struct timeval before, after;
>  +  osm_opensm_t *p_osm = p_oct->p_osm;
>  +    fabric_stats_t  fs;
>  +    struct timeval  before, after;
>      char *p_cmd;
>      memset(&fs, 0, sizeof(fs));
>  @@ -706,7 +780,7 @@ static void portstatus_parse(char **p_last, osm_opensm_t 
>  * p_osm, FILE * out)
>          } else if (strcmp(p_cmd, "router") == 0) {
>              fs.node_type_lim = IB_NODE_TYPE_ROUTER;
>          } else {
>  -            fprintf(out, "Node type not understood\n");
>  +            cio_printf(out, "Node type not understood\n");
>              help_portstatus(out, 1);
>              return;
>          }
>  @@ -723,58 +797,56 @@ static void portstatus_parse(char **p_last, 
>  osm_opensm_t * p_osm, FILE * out)
>      gettimeofday(&after, NULL);
>      /* report the stats */
>  -    fprintf(out, "\"%s\" port status:\n",
>  -        fs.node_type_lim ? ib_get_node_type_str(fs.
>  -                            node_type_lim) : "ALL");
>  -    fprintf(out,
>  -        "   %" PRIu64 " port(s) scanned on %" PRIu64
>  -        " nodes in %lu us\n", fs.total_ports, fs.total_nodes,
>  -        after.tv_usec - before.tv_usec);
>  +    cio_printf(out, "\"%s\" port status:\n",
>  +        fs.node_type_lim ?  ib_get_node_type_str(fs.node_type_lim) : 
>  "ALL");
>  +    cio_printf(out, "   %"PRIu64" port(s) scanned on %"PRIu64" nodes in %lu 
>  us\n",
>  +        fs.total_ports, fs.total_nodes, after.tv_usec - before.tv_usec);
>      if (fs.ports_down)
>  -        fprintf(out, "   %" PRIu64 " down\n", fs.ports_down);
>  +        cio_printf(out, "   %"PRIu64" down\n", fs.ports_down);
>      if (fs.ports_active)
>  -        fprintf(out, "   %" PRIu64 " active\n", fs.ports_active);
>  +        cio_printf(out, "   %"PRIu64" active\n", fs.ports_active);
>      if (fs.ports_1X)
>  -        fprintf(out, "   %" PRIu64 " at 1X\n", fs.ports_1X);
>  +        cio_printf(out, "   %"PRIu64" at 1X\n", fs.ports_1X);
>      if (fs.ports_4X)
>  -        fprintf(out, "   %" PRIu64 " at 4X\n", fs.ports_4X);
>  +        cio_printf(out, "   %"PRIu64" at 4X\n", fs.ports_4X);
>      if (fs.ports_8X)
>  -        fprintf(out, "   %" PRIu64 " at 8X\n", fs.ports_8X);
>  +        cio_printf(out, "   %"PRIu64" at 8X\n", fs.ports_8X);
>      if (fs.ports_12X)
>  -        fprintf(out, "   %" PRIu64 " at 12X\n", fs.ports_12X);
>  +        cio_printf(out, "   %"PRIu64" at 12X\n", fs.ports_12X);
>      if (fs.ports_sdr)
>  -        fprintf(out, "   %" PRIu64 " at 2.5 Gbps\n", fs.ports_sdr);
>  +        cio_printf(out, "   %"PRIu64" at 2.5 Gbps\n", fs.ports_sdr);
>      if (fs.ports_ddr)
>  -        fprintf(out, "   %" PRIu64 " at 5.0 Gbps\n", fs.ports_ddr);
>  +        cio_printf(out, "   %"PRIu64" at 5.0 Gbps\n", fs.ports_ddr);
>      if (fs.ports_qdr)
>  -        fprintf(out, "   %" PRIu64 " at 10.0 Gbps\n", fs.ports_qdr);
>  +        cio_printf(out, "   %"PRIu64" at 10.0 Gbps\n", fs.ports_qdr);
>      if (fs.ports_disabled + fs.ports_reduced_speed + fs.ports_reduced_width
>  -        > 0) {
>  -        fprintf(out, "\nPossible issues:\n");
>  +            > 0) {
>  +        cio_printf(out, "\nPossible issues:\n");
>      }
>      if (fs.ports_disabled) {
>  -        fprintf(out, "   %" PRIu64 " disabled\n", fs.ports_disabled);
>  +        cio_printf(out, "   %"PRIu64" disabled\n", fs.ports_disabled);
>          __print_port_report(out, fs.disabled_ports);
>      }
>      if (fs.ports_reduced_speed) {
>  -        fprintf(out, "   %" PRIu64 " with reduced speed\n",
>  +        cio_printf(out, "   %"PRIu64" with reduced speed\n",
>              fs.ports_reduced_speed);
>          __print_port_report(out, fs.reduced_speed_ports);
>      }
>      if (fs.ports_reduced_width) {
>  -        fprintf(out, "   %" PRIu64 " with reduced width\n",
>  +        cio_printf(out, "   %"PRIu64" with reduced width\n",
>              fs.ports_reduced_width);
>          __print_port_report(out, fs.reduced_width_ports);
>      }
>  -    fprintf(out, "\n");
>  +    cio_printf(out, "\n");
>  }
>  #ifdef ENABLE_OSM_PERF_MGR
>  -static void perfmgr_parse(char **p_last, osm_opensm_t * p_osm, FILE * out)
>  +static void perfmgr_parse(char **p_last, osm_console_thread_t *p_oct, CIO_t 
>  *out)
>  {
>  +  osm_opensm_t *p_osm = p_oct->p_osm;
>      char *p_cmd;
>      p_cmd = next_token(p_last);
>  @@ -803,309 +875,937 @@ static void perfmgr_parse(char **p_last, 
>  osm_opensm_t * p_osm, FILE * out)
>                  osm_perfmgr_set_sweep_time_s(&(p_osm->perfmgr),
>                                   time_s);
>              } else {
>  -                fprintf(out,
>  +                cio_printf(out,
>                      "sweep_time requires a time period (in seconds) to be 
>  specified\n");
>              }
>          } else {
>  -            fprintf(out, "\"%s\" option not found\n", p_cmd);
>  +            cio_printf(out, "\"%s\" option not found\n", p_cmd);
>          }
>      } else {
>  -        fprintf(out, "Performance Manager status:\n"
>  +        cio_printf(out, "Performance Manager status:\n"
>              "state                   : %s\n"
>              "sweep state             : %s\n"
>              "sweep time              : %us\n"
>  -            "outstanding queries/max : %d/%u\n"
>  -            "loaded event plugin     : %s\n",
>  +            "outstanding queries/max : %d/%u\n",
>              osm_perfmgr_get_state_str(&(p_osm->perfmgr)),
>              osm_perfmgr_get_sweep_state_str(&(p_osm->perfmgr)),
>              osm_perfmgr_get_sweep_time_s(&(p_osm->perfmgr)),
>              p_osm->perfmgr.outstanding_queries,
>  -            p_osm->perfmgr.max_outstanding_queries,
>  -            p_osm->perfmgr.event_plugin ?
>  -            p_osm->perfmgr.event_plugin->plugin_name : "NONE");
>  +            p_osm->perfmgr.max_outstanding_queries);
>      }
>  }
>  #endif                /* ENABLE_OSM_PERF_MGR */
>  -/* This is public to be able to close it on exit */
>  -void osm_console_close_socket(osm_opensm_t * p_osm)
>  +static void help_version(CIO_t *out, int detail)
>  {
>  -    if (p_osm->console.socket > 0) {
>  -        close(p_osm->console.in_fd);
>  -        p_osm->console.in_fd = -1;
>  -        p_osm->console.out_fd = -1;
>  -        p_osm->console.in = NULL;
>  -        p_osm->console.out = NULL;
>  -    }
>  +  cio_printf(out, "version -- print the OSM version\n");
>  }
>  -static void quit_parse(char **p_last, osm_opensm_t * p_osm, FILE * out)
>  +static void version_parse(char **p_last, osm_console_thread_t *p_oct, CIO_t 
>  *out)
>  {
>  -    osm_console_close_socket(p_osm);
>  +  cio_printf(out, "%s build %s %s\n", OSM_VERSION, __DATE__, __TIME__);
>  }
>  -static void help_version(FILE * out, int detail)
>  +/**********************************************************************
>  + * thread pool primitive: returns the thread structure to the pool, and
>  + * makes it available
>  + **********************************************************************/
>  +int free_console_thread(osm_console_thread_t *oct)
>  {
>  -    fprintf(out, "version -- print the OSM version\n");
>  +  // just clear the used flag, mark as available
>  +  oct->used = 0;
>  +  return 1;
>  +}
>  +
>  +/**********************************************************************
>  + * Cleans up the thread that was established for a connection.
>  + *   The connection should already be closed.  This method releases
>  + *   any resources and destroy the thread (done automagically??)
>  + *
>  + * refer to:  osm_console_thread and osm_console_thread_init
>  +**********************************************************************/
>  +int osm_console_thread_destroy(osm_console_thread_t *oct)
>  +{ +  free_console_thread(oct);
>  + +  // there are a few end cases that might need this (e.g. not completely 
>  init)
>  +  cio_close(getCIO(oct));
>  + +  return 0;
>  }
>  -static void version_parse(char **p_last, osm_opensm_t * p_osm, FILE * out)
>  +
>  +/**********************************************************************
>  + * Gracefully shut down the console connection, release resources
>  + *   refer to:  osm_console_init
>  + **********************************************************************/
>  +void osm_console_destroy(osm_console_thread_t *p_oct)
>  +{
>  +  osm_opensm_t *p_osm = p_oct->p_osm;
>  +  CIO_t *out = getCIO(p_oct);
>  +
>  +  osm_log(&(p_osm->log), OSM_LOG_INFO,
>  +      "osm_console_destroy: Console connection being closed:  %s (%s) 
>  s#%d\n", p_oct->client_hn,
>  +           p_oct->client_ip, out->fd);
>  +  fflush(p_osm->log.out_port);
>  +  cio_printf(out, "Closing this connection from osm_console_destroy\n");
>  + +  cio_close(out);
>  +  }
>  +
>  +/**********************************************************************
>  + * thread pool primitive: kills and disconnects connections.  If the
>  + * argument is a current thread, it will NOT be cleared (will be skipped)
>  + **********************************************************************/
>  +int kill_console_thread_pool(osm_console_thread_t* p_oct, osm_opensm_t 
>  *p_osm)
>  +{
>  +  // kill everything but my connection if p_oct is in the list
>  +  int i;
>  +  osm_console_thread_t* oct;
>  +  CIO_t *p_out = getCIO(p_oct);
>  +  CIO_t *out   = getCIO(p_oct);
>  +
>  +  // brute force this, don't use locks because don't want to get deadlocked
>  +//  cl_plock_acquire(&ThreadLock);
>  +  for(i = 0; i <  CIO_MAX_CONNECTS; ++i)
>  +  {
>  +    oct = &ConsoleThreadPool[i];
>  +    if((oct) && (oct->used) && (p_oct != oct))
>  +    {
>  +      cio_printf(p_out, " killing thread: %s\n", oct->name);
>  +      out = getCIO(oct);
>  +     +      // disconnect gracefully??
>  +      osm_log(&(p_osm->log), OSM_LOG_INFO,
>  +          "kill_console_thread_pool: %d (s#%d)\n", i, out->fd);
>  +     +      // return all the console resources
>  +      osm_console_destroy(oct);
>  +     +      // return all the thread and connection resources
>  +      osm_console_thread_destroy(oct);
>  +    }
>  +  }
>  +//  cl_plock_release(&ThreadLock);
>  +  return i;
>  +}
>  +
>  +/**********************************************************************
>  + * releases all of the resources used by all of the connections, by
>  + * closing sockets, freeing threads, etc..
>  + *
>  + * a good method for handling a kill signal
>  + **********************************************************************/
>  +int free_console_threads(osm_opensm_t *p_osm)
>  {
>  -    fprintf(out, "%s build %s %s\n", OSM_VERSION, __DATE__, __TIME__);
>  +  // just make sure everything is gone
>  +  int rtnval =  kill_console_thread_pool(NULL, p_osm);
>  +  return rtnval;
>  }
>  +
>  +/**********************************************************************
>  + * thread pool primitive: clears and initializes all the threads.  If the
>  + * argument is a current thread, it will NOT be cleared (will be skipped)
>  + **********************************************************************/
>  +int print_console_thread_pool(osm_console_thread_t* p_oct, osm_opensm_t 
>  *p_osm, CIO_t *out)

This function is not used.

>  +{
>  +  // show whats in use, and whats available
>  +
>  +  int i;
>  +  osm_console_thread_t* oct;
>  +
>  +  char *t_string = ctime(&(ServerTime.tv_sec));
>  +  t_string[strlen(t_string)-1]=0;     +  cio_printf(out, "OSM Server - Up 
>  since: %s, Users: %d, * = this connection\n", t_string, 
>  num_console_threads());
>  +
>  +  // (careful not to double lock .. num_console_threads() +  
>  cl_plock_acquire(&ThreadLock);
>  +
>  +  for(i = 0; i <  CIO_MAX_CONNECTS; ++i)
>  +  {
>  +    oct = &ConsoleThreadPool[i];
>  +    if((oct) && (oct->used))
>  +    {
>  +      if(p_oct == oct)
>  +        cio_printf(out, "*");
>  +      else
>  +        cio_printf(out, " ");
>  +      cio_printf(out,  "Thread:  %s [%d]\n", oct->name, oct->thread_num);
>  +      cio_printf(out, "  User:   %s, (%s)\n", oct->client_hn, 
>  oct->client_ip);
>  +      t_string = ctime(&(oct->connect_time.tv_sec));
>  +      t_string[strlen(t_string)-1]=0;     +      cio_printf(out, "  Since:  
>  %s\n", t_string);
>  +      cio_printf(out, "  Port:   %d\n", oct->port);
>  +      cio_printf(out, "  Socket: %d\n", oct->io.fd);
>  +      cio_printf(out, "  State:  %d\n", oct->state);
>  +    }
>  +  }
>  +  cl_plock_release(&ThreadLock);
>  +  return i;
>  +}
>  +
>  +/* close and free up resources used by socket */
>  +static void osm_console_deinit_socket(osm_opensm_t *p_osm)
>  +{
>  +  if (p_osm->console.socket > 0)
>  +  {
>  +    osm_log(&(p_osm->log), OSM_LOG_INFO,
>  +    "osm_console: Closing the primary (listening) socket connection 
>  (%d)\n", p_osm->console.in_fd);
>  +
>  +    close(p_osm->console.in_fd);
>  +    p_osm->console.in_fd = -1;
>  +    p_osm->console.out_fd = -1;
>  +    p_osm->console.in = NULL;
>  +    p_osm->console.out = NULL;
>  +  }
>  +}
>  +
>  +/* do everything necessary to gracefully turn off the console */
>  +void osm_console_server_destroy(osm_opensm_t *p_osm)
>  +{
>  +  /* make sure consoles are closed before stopping the main listener socket 
>  */
>  +  free_console_threads(p_osm);
>  + +  cl_plock_destroy(&ThreadLock);
>  + +  /* close the socket, listening for connections */
>  +  osm_console_deinit_socket(p_osm);
>  +}
>  +
>  +/* turns off the console, signature needs to match the parse_funciton() */
>  +static void quit_parse(char **p_last, osm_console_thread_t *p_oct, CIO_t 
>  *out)
>  +{
>  +  // set the "done" flag used by the isDone() method
>  +  p_oct->authorized = 0;  // temporarily use this as the done flag
>  +
>  +  // do other necessary things to clean up and turn off
>  +}
>  +
>  +
>  /* more parse routines go here */
>  static const struct command console_cmds[] = {
>  -    {"help", &help_command, &help_parse},
>  -    {"quit", &help_quit, &quit_parse},
>  -    {"loglevel", &help_loglevel, &loglevel_parse},
>  -    {"priority", &help_priority, &priority_parse},
>  -    {"resweep", &help_resweep, &resweep_parse},
>  -    {"status", &help_status, &status_parse},
>  -    {"logflush", &help_logflush, &logflush_parse},
>  -    {"querylid", &help_querylid, &querylid_parse},
>  -    {"portstatus", &help_portstatus, &portstatus_parse},
>  -    {"version", &help_version, &version_parse},
>  +    { "help",    &help_command,        &help_parse},
>  +    { OSM_QUIT_CMD,    &help_quit,        &quit_parse},
>  +    { "loglevel",    &help_loglevel,        &loglevel_parse},
>  +    { "priority",    &help_priority,        &priority_parse},
>  +    { "resweep",    &help_resweep,        &resweep_parse},
>  +    { "status",    &help_status,        &status_parse},
>  +    { "logflush",    &help_logflush,        &logflush_parse},
>  +    { "querylid",   &help_querylid,         &querylid_parse},
>  +    { "portstatus", &help_portstatus,       &portstatus_parse},
>  +  { "version", &help_version,       &version_parse},
>  #ifdef ENABLE_OSM_PERF_MGR
>      {"perfmgr", &help_perfmgr, &perfmgr_parse},
>  #endif                /* ENABLE_OSM_PERF_MGR */
>      {NULL, NULL, NULL}    /* end of array */
>  };
>  -static void parse_cmd_line(char *line, osm_opensm_t * p_osm)
>  -{
>  -    char *p_cmd, *p_last;
>  -    int i, found = 0;
>  -    FILE *out = p_osm->console.out;
>  -
>  -    while (isspace(*line))
>  -        line++;
>  -    if (!*line)
>  -        return;
>  -    /* find first token which is the command */
>  -    p_cmd = strtok_r(line, " \t\n\r", &p_last);
>  -    if (p_cmd) {
>  -        for (i = 0; console_cmds[i].name; i++) {
>  -            if (loop_command.on) {
>  -                if (!strcmp(p_cmd, "q")) {
>  -                    loop_command.on = 0;
>  -                }
>  -                found = 1;
>  -                break;
>  -            }
>  -            if (!strcmp(p_cmd, console_cmds[i].name)) {
>  -                found = 1;
>  -                console_cmds[i].parse_function(&p_last, p_osm,
>  -                                   out);
>  -                break;
>  -            }
>  -        }
>  -        if (!found) {
>  -            fprintf(out, "%s : Command not found\n\n", p_cmd);
>  -            help_command(out, 0);
>  -        }
>  -    } else {
>  -        fprintf(out, "Error parsing command line: `%s'\n", line);
>  -    }
>  -    if (loop_command.on) {
>  -        fprintf(out, "use \"q<ret>\" to quit loop\n");
>  -        fflush(out);
>  -    }
>  +static void parse_cmd_line(char *line, osm_console_thread_t *oct)
>  +{
>  +  char *p_cmd, *p_last;
>  +  int i, found = 0;
>  +  CIO_t *out = getCIO(oct);
>  + +  while (isspace(*line))
>  +    line++;
>  +  if (!*line)
>  +    return;
>  +
>  +  /* find first token which is the command */
>  +  p_cmd = strtok_r(line, " \t\n\r", &p_last);
>  +  if (p_cmd) {
>  +    for (i = 0; console_cmds[i].name; i++) {
>  +      if (oct->loop_command.on ) {
>  +        if (!strcmp(p_cmd, "q")) {
>  +          oct->loop_command.on = 0;
>  +        }
>  +        found = 1;
>  +        break;
>  +      }
>  +      if (!strcmp(p_cmd, console_cmds[i].name)) {
>  +        found = 1;
>  +        console_cmds[i].parse_function(&p_last, oct, out);
>  +        break;
>  +      }
>  +    }
>  +    if (!found) {
>  +      cio_printf(out, "%s : Command not found\n\n", p_cmd);
>  +      help_command(out, 0);
>  +    }
>  +  } else {
>  +    cio_printf(out, "Error parsing command line: `%s'\n", line);
>  +  }
>  }
>  -void osm_console_prompt(FILE * out)
>  +void osm_console_prompt(CIO_t *out, int loop_prompt)
>  {
>      if (out) {
>  -        fprintf(out, "OpenSM %s", OSM_COMMAND_PROMPT);
>  -        fflush(out);
>  +      if(loop_prompt)
>  +        cio_printf(out, "use \"q<ret>\" to quit loop\n");
>  +      else
>  +      cio_printf(out, "OpenSM %s", OSM_COMMAND_PROMPT);
>  +        cio_flush(out);
>      }
>  }
>  -void osm_console_init(osm_subn_opt_t * opt, osm_opensm_t * p_osm)
>  +/* open and setup socket connection */
>  +static void osm_console_init_socket(osm_opensm_t *p_osm, uint16_t 
>  console_port, char* console_type)
>  {
>  -    p_osm->console.socket = -1;
>  -    /* set up the file descriptors for the console */
>  -    if (strcmp(opt->console, OSM_LOCAL_CONSOLE) == 0) {
>  -        p_osm->console.in = stdin;
>  -        p_osm->console.out = stdout;
>  -        p_osm->console.in_fd = fileno(stdin);
>  -        p_osm->console.out_fd = fileno(stdout);
>  -
>  -        osm_console_prompt(p_osm->console.out);
>  #ifdef ENABLE_OSM_CONSOLE_SOCKET
>  -    } else if (strcmp(opt->console, OSM_REMOTE_CONSOLE) == 0
>  -           || strcmp(opt->console, OSM_LOOPBACK_CONSOLE) == 0) {
>  -        struct sockaddr_in sin;
>  -        int optval = 1;
>  -
>  -        if ((p_osm->console.socket =
>  -             socket(AF_INET, SOCK_STREAM, 0)) < 0) {
>  -            osm_log(&(p_osm->log), OSM_LOG_ERROR,
>  -                "osm_console_init: ERR 4B01: Failed to open console socket: 
>  %s\n",
>  -                strerror(errno));
>  -            return;
>  -        }
>  -        setsockopt(p_osm->console.socket, SOL_SOCKET, SO_REUSEADDR,
>  -               &optval, sizeof(optval));
>  -        sin.sin_family = AF_INET;
>  -        sin.sin_port = htons(opt->console_port);
>  -        if (strcmp(opt->console, OSM_REMOTE_CONSOLE) == 0)
>  -            sin.sin_addr.s_addr = htonl(INADDR_ANY);
>  -        else
>  -            sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
>  -        if (bind(p_osm->console.socket, &sin, sizeof(sin)) < 0) {
>  -            osm_log(&(p_osm->log), OSM_LOG_ERROR,
>  -                "osm_console_init: ERR 4B02: Failed to bind console socket: 
>  %s\n",
>  -                strerror(errno));
>  -            return;
>  -        }
>  -        if (listen(p_osm->console.socket, 1) < 0) {
>  -            osm_log(&(p_osm->log), OSM_LOG_ERROR,
>  -                "osm_console_init: ERR 4B03: Failed to listen on socket: 
>  %s\n",
>  -                strerror(errno));
>  -            return;
>  -        }
>  -        signal(SIGPIPE, SIG_IGN);    /* protect ourselves from closed pipes 
>  */
>  -        p_osm->console.in = NULL;
>  -        p_osm->console.out = NULL;
>  -        p_osm->console.in_fd = -1;
>  -        p_osm->console.out_fd = -1;
>  -        osm_log(&(p_osm->log), OSM_LOG_INFO,
>  -            "osm_console_init: Console listening on port %d\n",
>  -            opt->console_port);
>  +  struct sockaddr_in sin;
>  +  int optval = 1;
>  + +  osm_log(&(p_osm->log), OSM_LOG_INFO, "osm_console_init_socket: 
>  Initializing the socket: %d\n", console_port);
>  + +  if ((p_osm->console.socket = socket(AF_INET, SOCK_STREAM, 0)) < 0)
>  +  {
>  +    osm_log(&(p_osm->log), OSM_LOG_ERROR, "osm_console_init_socket: ERR 
>  4B01: Failed to open console socket: %s\n", strerror(errno));
>  +    return;
>  +  }
>  +  setsockopt(p_osm->console.socket, SOL_SOCKET, SO_REUSEADDR, &optval, 
>  sizeof(optval));
>  +  sin.sin_family = AF_INET;
>  +  sin.sin_port = htons(console_port);
>  +
>  +  // loopback or ...
>  +  if(is_loopback(console_type))
>  +    sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
>  +  else
>  +    sin.sin_addr.s_addr = htonl(INADDR_ANY);
>  +  if (bind(p_osm->console.socket, &sin, sizeof(sin))< 0)
>  +  {
>  +    osm_log(&(p_osm->log), OSM_LOG_ERROR, "osm_console_init_socket: ERR 
>  4B02: Failed to bind console socket: %s\n", strerror(errno));
>  +    return;
>  +  }
>  +  if (listen(p_osm->console.socket, 2)< 0)
>  +  {
>  +    osm_log(&(p_osm->log), OSM_LOG_ERROR, "osm_console_init_socket: ERR 
>  4B03: Failed to listen on socket: %s\n", strerror(errno));
>  +    return;
>  +  }
>  +
>  +  signal(SIGPIPE, SIG_IGN); /* protect ourselves from closed pipes */
>  +  p_osm->console.in = NULL;
>  +  p_osm->console.out = NULL;
>  +  p_osm->console.in_fd = -1;
>  +  p_osm->console.out_fd = -1;
>  +  osm_log(&(p_osm->log), OSM_LOG_INFO, "osm_console_init_socket: Console 
>  listening on port %d\n", console_port);
>  #endif
>  -    }
>  }
>  -#ifdef ENABLE_OSM_CONSOLE_SOCKET
>  -static void handle_osm_connection(osm_opensm_t * p_osm, int new_fd,
>  -                  char *client_ip, char *client_hn)
>  +/**********************************************************************
>  + * thread pool primitive: gets the next available thread structure from
>  + * the pool.
>  + *
>  + * refer to free_console_thread()
>  + **********************************************************************/
>  +osm_console_thread_t* new_console_thread(void)
>  {
>  -    char *p_line;
>  -    size_t len;
>  -    ssize_t n;
>  -
>  -    if (p_osm->console.in_fd >= 0) {
>  -        FILE *file = fdopen(new_fd, "w+");
>  -
>  -        fprintf(file, "OpenSM Console connection already in use\n"
>  -            "   kill other session (y/n)? ");
>  -        fflush(file);
>  -        p_line = NULL;
>  -        n = getline(&p_line, &len, file);
>  -        if (n > 0 && (p_line[0] == 'y' || p_line[0] == 'Y')) {
>  -            osm_console_close_socket(p_osm);
>  -        } else {
>  -            close(new_fd);
>  -            return;
>  -        }
>  -    }
>  -    p_osm->console.in_fd = new_fd;
>  -    p_osm->console.out_fd = p_osm->console.in_fd;
>  -    p_osm->console.in = fdopen(p_osm->console.in_fd, "w+");
>  -    p_osm->console.out = p_osm->console.in;
>  -    osm_console_prompt(p_osm->console.out);
>  -    osm_log(&(p_osm->log), OSM_LOG_INFO,
>  -        "osm_console_init: Console connection accepted: %s (%s)\n",
>  -        client_hn, client_ip);
>  +  // return the next available thread from the pool
>  +  //  just iterate through..
>  +
>  +  int i;
>  +  osm_console_thread_t* next = NULL;
>  + +  cl_plock_acquire(&ThreadLock);
>  +  for(i = 0; i <  CIO_MAX_CONNECTS; ++i)
>  +  {
>  +    next = &ConsoleThreadPool[i];
>  +    if(next->used == 0)
>  +      break;
>  +  }
>  + +  if(i >= CIO_MAX_CONNECTS)
>  +    next = NULL;  // full
>  +  else
>  +  {
>  +    // immediately mark this as NOT available
>  +    next->used = 1;
>  +    next->thread_num = ++cio_thread_counter;
>  +    gettimeofday(&(next->connect_time), NULL);   +  }
>  +  cl_plock_release(&ThreadLock);
>  + +  return next;
>  }
>  -static int connection_ok(char *client_ip, char *client_hn)
>  +/**********************************************************************
>  + * thread pool primitive: clears and initializes all the threads.  If the
>  + * argument is a current thread, it will NOT be cleared (will be skipped)
>  + **********************************************************************/
>  +int init_console_thread_pool(osm_console_thread_t* p_oct, osm_subn_opt_t 
>  *opt, osm_opensm_t *p_osm)
>  {
>  -    return (hosts_ctl
>  -        (OSM_DAEMON_NAME, client_hn, client_ip, "STRING_UNKNOWN"));
>  +  // initialize
>  +
>  +  int i;
>  +  osm_console_thread_t* oct;
>  + +  cl_plock_acquire(&ThreadLock);
>  +  for(i = 0; i <  CIO_MAX_CONNECTS; ++i)
>  +  {
>  +    oct = &ConsoleThreadPool[i];
>  +    if(p_oct == NULL || p_oct != oct)
>  +    {
>  +      oct->used        = 0;
>  +      oct->thread_num  = -1;
>  +      oct->authorized  = 0;
>  +      oct->port        = CIO_CONNECTION_PORT;
>  +      oct->io.fd       = -1;
>  +      oct->state       = 0;
>  +      oct->p_osm       = p_osm;
>  +      if(opt != NULL)
>  +      {
>  +        oct->port      = opt->console_port;
>  +        strncpy(oct->name, opt->console, CIO_INFO_SIZE);
>  +      }
>  +    }
>  +  }
>  +  cl_plock_release(&ThreadLock);
>  +  return i;
>  }
>  -#endif
>  -void osm_console(osm_opensm_t * p_osm)
>  +void osm_console_server_init(osm_subn_opt_t *opt, osm_opensm_t *p_osm)
>  {
>  -    struct pollfd pollfd[2];
>  -    char *p_line;
>  -    size_t len;
>  -    ssize_t n;
>  -    struct pollfd *fds;
>  -    nfds_t nfds;
>  -
>  -    pollfd[0].fd = p_osm->console.socket;
>  -    pollfd[0].events = POLLIN;
>  -    pollfd[0].revents = 0;
>  -
>  -    pollfd[1].fd = p_osm->console.in_fd;
>  -    pollfd[1].events = POLLIN;
>  -    pollfd[1].revents = 0;
>  -
>  -    fds = p_osm->console.socket < 0 ? &pollfd[1] : pollfd;
>  -    nfds = p_osm->console.socket < 0 || pollfd[1].fd < 0 ? 1 : 2;
>  -
>  -    if (loop_command.on && loop_command_check_time() &&
>  -        loop_command.loop_function) {
>  -        if (p_osm->console.out) {
>  -            loop_command.loop_function(p_osm, p_osm->console.out);
>  -            fflush(p_osm->console.out);
>  -        } else {
>  -            loop_command.on = 0;
>  -        }
>  -    }
>  +  int status = 0;
>  +
>  +  cl_plock_construct(&ThreadLock);
>  +  status = cl_plock_init(&ThreadLock);
>  +  if (status != IB_SUCCESS)
>  +    osm_log(&(p_osm->log), OSM_LOG_ERROR, "osm_console_server_init: lock 
>  initialization error\n");
>  + +  init_console_thread_pool(NULL, opt, p_osm);
>  + +  gettimeofday(&ServerTime, NULL);  // start time
>  + +  p_osm->console.socket = -1;
>  +
>  +  /* set up the file descriptors for the console */
>  +  if (strcmp(opt->console, OSM_LOCAL_CONSOLE)== 0)
>  +  {
>  +    p_osm->console.in = stdin;
>  +    p_osm->console.out = stdout;
>  +    p_osm->console.in_fd = fileno(stdin);
>  +    p_osm->console.out_fd = fileno(stdout);
>  +  }
>  +  else if (is_remote(opt->console))
>  +  {
>  +    osm_console_init_socket(p_osm, opt->console_port, opt->console);
>  +  }
>  +  // TODO - other types of "console" connections here
>  +}
>  -    if (poll(fds, nfds, 1000) <= 0)
>  -        return;
>  +/**********************************************************************
>  + * Main Loop Thread.
>  + *
>  + * Continuously loop on this command until turned off
>  + **********************************************************************/
>  +void osm_loop_thread(void *p_ptr)
>  +{
>  +  osm_console_thread_t *oct = ( osm_console_thread_t * ) p_ptr;
>  +  CIO_t *p_io = getCIO(oct);
>  + +  oct->loop_command.running = 1;
>  +  while (oct->loop_command.on && oct->loop_command.loop_function)
>  +  {
>  +    if (p_io->out)
>  +    {
>  +      // dwell here
>  +      cl_thread_suspend(oct->loop_command.delay_s * 1000);
>  +      oct->loop_command.loop_function(oct->p_osm, p_io);
>  +     +      // send the cmd prompt
>  +      osm_console_prompt(p_io, oct->loop_command.on);    +      
>  cio_flush(p_io);
>  +    }
>  +    else
>  +    {
>  +      oct->loop_command.on = 0;
>  +    }
>  +  }
>  +  oct->loop_command.running = 0;
>  +  return;
>  +}
>  +/**********************************************************************
>  + * Do authentication & authorization check
>  + **********************************************************************/
>  +static int is_authorized(osm_console_thread_t *p_oct)
>  +{
>  #ifdef ENABLE_OSM_CONSOLE_SOCKET
>  -    if (pollfd[0].revents & POLLIN) {
>  -        int new_fd = 0;
>  -        struct sockaddr_in sin;
>  -        socklen_t len = sizeof(sin);
>  -        char client_ip[64];
>  -        char client_hn[128];
>  -        struct hostent *hent;
>  -        if ((new_fd = accept(p_osm->console.socket, &sin, &len)) < 0) {
>  -            osm_log(&(p_osm->log), OSM_LOG_ERROR,
>  -                "osm_console: ERR 4B04: Failed to accept console socket: 
>  %s\n",
>  -                strerror(errno));
>  -            p_osm->console.in_fd = -1;
>  -            return;
>  -        }
>  -        if (inet_ntop
>  -            (AF_INET, &sin.sin_addr, client_ip,
>  -             sizeof(client_ip)) == NULL) {
>  -            snprintf(client_ip, 64, "STRING_UNKNOWN");
>  -        }
>  -        if ((hent = gethostbyaddr((const char *)&sin.sin_addr,
>  -                      sizeof(struct in_addr),
>  -                      AF_INET)) == NULL) {
>  -            snprintf(client_hn, 128, "STRING_UNKNOWN");
>  -        } else {
>  -            snprintf(client_hn, 128, "%s", hent->h_name);
>  -        }
>  -        if (connection_ok(client_ip, client_hn)) {
>  -            handle_osm_connection(p_osm, new_fd, client_ip,
>  -                          client_hn);
>  -        } else {
>  -            osm_log(&(p_osm->log), OSM_LOG_ERROR,
>  -                "osm_console: ERR 4B05: Console connection denied: %s 
>  (%s)\n",
>  -                client_hn, client_ip);
>  -            close(new_fd);
>  -        }
>  -        return;
>  -    }
>  +  ////  oct->authorized = pam_authorize(pTs);
>  +  p_oct->authorized = !is_remote(p_oct->client_type) ||
>  +                       hosts_ctl(OSM_DAEMON_NAME, p_oct->client_hn, 
>  p_oct->client_ip, "STRING_UNKNOWN");
>  +#else
>  +  p_oct->authorized = 1;
>  #endif
>  +  return p_oct->authorized;
>  +}
>  -    if (pollfd[1].revents & POLLIN) {
>  -        p_line = NULL;
>  -        /* Get input line */
>  -        n = getline(&p_line, &len, p_osm->console.in);
>  -        if (n > 0) {
>  -            /* Parse and act on input */
>  -            parse_cmd_line(p_line, p_osm);
>  -            if (!loop_command.on) {
>  -                osm_console_prompt(p_osm->console.out);
>  -            }
>  -        } else
>  -            osm_console_close_socket(p_osm);
>  -        if (p_line)
>  -            free(p_line);
>  -    }
>  +/*
>  + * determine if the connection should be closed
>  + */
>  +static int is_done(osm_console_thread_t *oct)
>  +{
>  +  int done = 0;  // set to 1 when finished
>  + +  /* Look for a condition that signals the connection should be closed */
>  +  if (!(oct->authorized) || !strcmp(oct->in_buff, OSM_QUIT_CMD) || 
>  osm_exit_flag)
>  +  {
>  +    done = 1;
>  +  }
>  +  return (done);
>  +}
>  +
>  +/*
>  + * handle basic output to the client
>  + *
>  + * this includes results from a command, error information
>  + * or any appropriate feedback
>  + */
>  +static int output(osm_console_thread_t *oct)
>  +{
>  +  CIO_t *out = getCIO(oct);
>  + +  // send the output buffer to the client
>  +  cio_printf(out, oct->out_buff);
>  +  cio_flush(out);
>  + +  // clear the output buffer??
>  +  oct->out_buff[0] = 0;
>  +
>  +  // send the cmd prompt
>  +  if(!oct->loop_command.on)
>  +    osm_console_prompt(out, 0);
>  + +  return (is_done(oct));
>  +}
>  +
>  +/*
>  + * handle basic input from the socket
>  + */
>  +static int input(osm_console_thread_t *oct)
>  +{
>  +  char *p_line = NULL;
>  +  size_t len;
>  +  ssize_t n;
>  +  CIO_t *p_io = getCIO(oct);
>  + +  // if we are in a loop command, the don't block
>  +  if(oct->loop_command.on && !cio_poll(p_io, 1000))
>  +    return 0;
>  +
>  +  /* Get input line */
>  +  n = cio_getline(&p_line, &len, p_io);
>  +  if (n > 0)
>  +  {
>  +    // got something, so copy it to the input buffer
>  +    sprintf(oct->in_buff, "%s", p_line);     +
>  +    if(p_line)
>  +     free(p_line);
>  +  }
>  + + return (0);
>  +}
>  +
>  +/*
>  + * process the command in the input buffer -
>  + *    take action, produce results, copy to output buffer
>  + */
>  +static int commands(osm_console_thread_t *oct)
>  +{
>  +  osm_opensm_t *p_osm = oct->p_osm;
>  +  +   ib_api_status_t status = IB_INSUFFICIENT_RESOURCES;
>  +
>  +   parse_cmd_line(oct->in_buff, oct);
>  +  +   /* if parsed and executed then clear the input buffer
>  +    */
>  +   oct->in_buff[0] = 0;
>  +  +   /* special case, only allow one loop command
>  +    */
>  +   if(!oct->loop_command.running && oct->loop_command.on && 
>  oct->loop_command.loop_function)
>  +   {
>  +     status = cl_thread_init(&oct->loop_command.loopThread, 
>  osm_loop_thread, oct, "Loop command");   +     if (status != IB_SUCCESS)
>  +     {
>  +       // something bad
>  +       osm_log(&(p_osm->log), OSM_LOG_ERROR,
>  +           "commands: Couldn't create a thread for the loop command!\n");
>  +       return -1;
>  +     }
>  +   }
>  +   return (0);
>  +}
>  +
>  +/**********************************************************************
>  + * Initialization and configuration of the console connection.
>  + * (security & authorization, plus some bookkeeping)
>  + *
>  + * returns 1 if okay
>  + *         0 if not authorized
>  + *        -1 if too many connections
>  + *        -2 if error??
>  + **********************************************************************/
>  +int osm_console_init(osm_console_thread_t *p_oct)
>  +{
>  +  // the first opportunity to do thread specific actions
>  + +  int status = 0; // not authorized
>  +  int max_connects_exceeded = (num_console_threads() >= CIO_MAX_CONNECTS);
>  +
>  +  osm_opensm_t *p_osm = p_oct->p_osm;
>  +  CIO_t *p_io = getCIO(p_oct);
>  +
>  +  // check for authorization
>  +  if(is_authorized(p_oct))
>  +  {
>  +    // check for available connections (too many?)
>  +    if (!max_connects_exceeded)
>  +     {
>  +       cio_open(p_io);
>  +
>  +       osm_log(&(p_osm->log), OSM_LOG_INFO, "osm_console_init: Console 
>  connection accepted: %s (%s) s#%d\n", p_oct->client_hn,
>  +           p_oct->client_ip, p_io->fd);
>  +       status = 1;
>  +     }
>  +     else
>  +     {
>  +       osm_log(&(p_osm->log), OSM_LOG_ERROR, "osm_console_init: ERR 4B06: 
>  No available connections: %s (%s) t#%d\n", p_oct->client_hn,
>  +           p_oct->client_ip, num_console_threads());
>  +       status = -1;
>  +     }   +  }
>  +  else
>  +  {
>  +    osm_log(&(p_osm->log), OSM_LOG_ERROR, "osm_console_init: ERR 4B05: 
>  Console connection denied: %s (%s)\n", p_oct->client_hn,
>  +        p_oct->client_ip);
>  +    status = 0;   +  }
>  + +  fflush(p_osm->log.out_port);
>  +  return status;
>  +}
>  +
>  +/**********************************************************************
>  + * The console I/O and command loop
>  + *   refer to:  osm_console_init and osm_console_destroy
>  + **********************************************************************/
>  +void osm_console(osm_console_thread_t *oct)
>  +{
>  +  cl_thread_suspend(100);  // wait for other threads to initialize
>  + +  // provide feedback from the server (probably from a previous command)
>  +  while(!output(oct))
>  +  {
>  +    // read the socket
>  +    input(oct);
>  +   +    // process or act on the input
>  +    commands(oct);
>  +  }
>  +  // final methods??
>  }
>  +
>  +/**********************************************************************
>  + * Main Console Thread.
>  + *
>  + * Finish setting up the connection ( secure & authorized) and misc config
>  + *
>  + * Loop continuously in the osm_console method.
>  + *
>  + * Clean up, and gracefully exit when done
>  + **********************************************************************/
>  +void osm_console_thread(void *p_ptr)
>  +{
>  +  osm_console_thread_t *p_oct = ( osm_console_thread_t * ) p_ptr;
>  + +  /* Finish setting up the connection (secure & authorized) and misc 
>  config */
>  +  if(osm_console_init(p_oct) == 1)
>  +  {
>  +    // do all i/o and commands until done
>  +    osm_console(p_oct);
>  +   +    // done, so close down the console gracefully
>  +    osm_console_destroy(p_oct);
>  +  } + +  // nothing left to do but destroy our own thread, return to pool
>  +  osm_console_thread_destroy(p_oct);
>  +  return; +}
>  +
>  +/* Prepare to launch the console by encapsulating all the necessary data in 
>  a thread
>  + * safe data structure.
>  + *
>  + * Support for single (local) or multiple (socket) threads.
>  + *
>  + * initialize the console data structure for a thread, and then..
>  + *   if socket
>  + *      create the thread
>  + *   else
>  + *      run inline
>  + *
>  + * refer to:  osm_console_thread and osm_console_thread_destroy
>  + *
>  + */
>  +int osm_console_thread_init(int socket, struct sockaddr_in *sin, 
>  osm_subn_opt_t *p_opt, osm_opensm_t *p_osm)
>  +{
>  +  static int n_local = 0;
>  +  osm_console_thread_t *oct;        // see free_console_thread() !!
>  +  ib_api_status_t status = IB_INSUFFICIENT_RESOURCES;
>  +
>  +  // have we used up all available connections?
>  +  if ((!is_remote(p_opt->console) && n_local) || ((oct = 
>  new_console_thread())== NULL))
>  +  {
>  +    if(n_local)
>  +      cl_thread_suspend( 100000);  // denied, dwell here before trying 
>  again.
>  +    else
>  +      osm_log(&(p_osm->log), OSM_LOG_ERROR,
>  +          "osm_console_thread_init: Maximum number of connections exceeded, 
>  connection denied (%d)\n",
>  +          num_console_threads());
>  +    return status;
>  +  }
>  + +  if(!is_remote(p_opt->console))
>  +      n_local++;  // only one local connection...
>  +
>  +  /* fill in the osm_console_thread_t structure (can't be NULL) */
>  +  oct->authorized = 0;
>  +  oct->state = 0;
>  +  oct->p_osm = p_osm;
>  +  oct->io.fd = socket;
>  +  oct->port = p_opt->console_port;
>  +  snprintf(oct->client_type, CIO_NOTE_SIZE, p_opt->console);
>  + +#ifdef ENABLE_OSM_CONSOLE_SOCKET
>  +  /* get then name and ip of the client (console connection) */
>  +  if(is_remote(oct->client_type))
>  +  {
>  +    /* get the clients ip address */
>  +    if (inet_ntop(AF_INET, &sin->sin_addr, oct->client_ip, 
>  sizeof(oct->client_ip))== NULL)
>  +    {
>  +      snprintf(oct->client_ip, CIO_NOTE_SIZE, "STRING_UNKNOWN");
>  +    }
>  +   +    /* get the clients host name */
>  +    struct hostent *hent;
>  +    if ((hent = gethostbyaddr((const char *)&sin->sin_addr, sizeof(struct 
>  in_addr), AF_INET)) == NULL)
>  +    {
>  +      snprintf(oct->client_hn, CIO_INFO_SIZE, "STRING_UNKNOWN");
>  +    }
>  +    else
>  +    {
>  +      snprintf(oct->client_hn, CIO_INFO_SIZE, "%s", hent->h_name);
>  +    }   +  } +  else
>  +#endif   +  {
>  +    if(gethostname(oct->client_hn, CIO_INFO_SIZE))
>  +    {
>  +      snprintf(oct->client_hn, CIO_INFO_SIZE, "localhost");
>  +      snprintf(oct->client_ip, CIO_NOTE_SIZE, "localhost");
>  +    }
>  +    else
>  +      snprintf(oct->client_ip, CIO_NOTE_SIZE, oct->client_hn);   +  }
>  +   +
>  +  // create a name for the thread, based on the connection
>  +  snprintf(oct->name, CIO_INFO_SIZE, "%s %d", OSM_CONSOLE_NAME, 
>  oct->io.fd);
>  +
>  +  // ***** Finally, create a new thread for this connection ******
>  +  status = cl_thread_init(&oct->consoleThread, osm_console_thread, oct, 
>  oct->name);   +  if (status != IB_SUCCESS)
>  +  {
>  +    // something bad
>  +    osm_log(&(p_osm->log), OSM_LOG_ERROR,
>  +        "osm_console_thread_init: Couldn't create a thread for the 
>  socket!\n");
>  +
>  +    // free up the thread, wasn't actually used   +    
>  osm_console_thread_destroy(oct);
>  +    return -1;
>  +  }
>  +  return 0;
>  +}
>  +
>  +
>  +/* Multi-threaded service to handle zero or more osm_consoles
>  + *
>  + * Typically the OSM runs as a daemon process, with zero
>  + *   consoles.  Occationally it is necessary to remotely connect
>  + *   to the OSM through a console connection.
>  + *
>  + * Allow one Master remote console and many Slaves.
>  + *
>  + * Provide a mechanism to release and assume Master role.
>  + *
>  + */
>  +int osm_console_server(osm_subn_opt_t *p_opt, osm_opensm_t *p_osm)
>  +{
>  +  struct sockaddr_in sin;
>  +  int s = 0;
>  +
>  +  /* don't enter this code section, if the exit flag is true */
>  +  if (!osm_exit_flag)
>  +  {
>  +    // handle IO from local or remote console
>  +    // blocks here until a client tries to connect
>  +
>  +    /*
>  +     * this version is supposed to block
>  +     *
>  +     * the block is released when a connection occurs, which causes a new
>  +     * thread to be spawned to handle the connection.  The new thread 
>  cleans
>  +     * up after itself.
>  +     *
>  +     * return only happens after a successful connection has been 
>  established,
>  +     * and needs to be prepared for another connection.
>  +     */
>  +#ifdef ENABLE_OSM_CONSOLE_SOCKET
>  +    socklen_t len = sizeof(sin);
>  +    if (is_remote(p_opt->console) && ((s = accept(p_osm->console.socket, 
>  &sin, &len)) < 0))
>  +    {
>  +      // kill sig can cause this... which would be normal during a shutdown
>  +      osm_log(&(p_osm->log), OSM_LOG_ERROR, "osm_console_server: not 
>  accepting socket connections\n");
>  +      return -1;
>  +    }
>  +    else
>  +#endif
>  +      // create a thread to handle the i/o on this connection
>  +      osm_console_thread_init(s, &sin, p_opt, p_osm);
>  +  }
>  +  else
>  +    free_console_threads(p_osm);  // clean up
>  +  return s;
>  +}
>  +
>  +
>  +/**********************************************************************
>  + * Function Name:
>  + *             cio_vprintf
>  + *
>  + * This routine formats a message and uses a Stream IO abstraction to 
>  determine
>  + * how and where to write the message out (stdout, socket, ssl, etc.)
>  + *
>  + * Side Effects:
>  + *             Unknown, uses vsprintf and variable arguments.  Possible 
>  stack problems.
>  + *
>  + * cio         pointer to the Connection IO data structure - an IO Stream 
>  abstraction
>  + *
>  + * format      A string literal that describes the desired text and 
>  formatting.  See printf().
>  + *
>  + * args        A variable argument list, of the type available between a 
>  va_start() and
>  + *             va_end() block.
>  + *
>  + * Always returns 0
>  + 
>  ******************************************************************************/
>  +
>  + int cio_vprintf( CIO_t *cio, const char *format, va_list args)
>  + {
>  +   char msg_buffer[CIO_BUFSIZE];
>  +
>  +   // create the formatted string and place it in the local string buffer
>  +   vsprintf(msg_buffer, format, args);
>  +  +   // send it out the proper I/O channel
>  +   fprintf(cio->out, msg_buffer);
>  +
>  +   return 0;
>  + }
>  +
>  +/******************************************************************************
>  + * Function Name:
>  + *             cio_printf
>  + *
>  + * This is an abstract form of the standard fprintf() routine.  It can be 
>  used
>  + * in an identical manner, with the exception of the first argument that 
>  needs
>  + * to be the Connection IO abstraction, rather than a FILE.
>  + *
>  + * Side Effects:
>  + *             Unknown, uses vsprintf and variable arguments.  Possible 
>  stack problems.
>  + *
>  + * cio         pointer to the Connection IO data structure - an IO Stream 
>  abstraction
>  + *
>  + * format      A string literal that describes the desired text and 
>  formatting.  See printf().
>  + *
>  + * args        A variable argument list, of the type available between a 
>  va_start() and
>  + *             va_end() block.
>  + *
>  + * Always returns 0, from cio_vprintf()
>  + 
>  ******************************************************************************/
>  +
>  + int cio_printf( CIO_t *cio, const char *format, ...)
>  + {
>  +   int returnval = 0;
>  +   va_list args;
>  +  +   // Sink Filter or Message Filter.  Does it get printed??
>  +   if(1)
>  +   {
>  +     va_start(args, format);
>  +     returnval = cio_vprintf(cio, format, args);
>  +     va_end(args);
>  +   }
>  +   return returnval;
>  + }
>  +
>  + int cio_flush( CIO_t *cio)
>  + {
>  +   int returnval = fflush(cio->out);
>  +  +   return returnval;
>  + }
>  +
>  + int cio_getline( char **lineptr, size_t *n, CIO_t *cio)
>  + {
>  +   int returnval = getline(lineptr, n, cio->in);
>  +  +   return returnval;
>  + }
>  +
>  + int cio_open( CIO_t *cio)
>  + {
>  +   // returns zero, if opened fine, -1 otherwise
>  +  +   struct pollfd *pd = (struct pollfd* )malloc(sizeof(struct pollfd));
>  +   if (pd == NULL)
>  +     return -1;  // should not happen
>  +  +   cio->in = fdopen(cio->fd, "w+");
>  +   cio->out = cio->in;
>  +   cio->err = cio->in;
>  +  +   cio->pfd = pd;
>  +   cio->pfd[0].fd = cio->fd;
>  +   cio->pfd[0].events = POLLIN;
>  +   cio->pfd[0].revents = 0;
>  +  +   return (cio->in == NULL) ? -1 : 0;
>  + }
>  +
>  + int cio_close( CIO_t *cio)
>  + {
>  +   int rtnval = -1;
>  +   if(cio && (cio->fd > 0))
>  +   {
>  +     free(cio->pfd);
>  +     rtnval = close(cio->fd);
>  +   }
>  +   cio->fd = 0;
>  +   return rtnval;
>  + }
>  +
>  + /* return true if input available */
>  + int cio_poll(CIO_t *cio, int timeout)
>  + {
>  +   // if timeout is less than 1, return true, alw
>  +   if(timeout < 1)
>  +     return 1;
>  +   return (poll(cio->pfd, 1, timeout) > 0);
>  + }    

It is not clear for me why most of those wrapper functions are needed
at all. And how really so big comment about *_printf() usage is helpful.

Sasha



More information about the general mailing list