#!/bin/sh
#
# meta: Sun Enterprise Server check - BB external script test
#
#####  Purpose is to report back to a central server, all Solaris
#####     DiskSuite RAID software faults.
#####
#
# version 1.0
# version 2.0 - properly uses $THIS_HOST instead of $MACHINE due to fqdn using
#                 comma in name
#               removed all direct program calls in favor of env vars in bbsys.*
#               changed /tmp to $BBTMP
#                       meta to $TEST
#                       $THIS_HOST to $MACHINE
#               removed code that checked for the existance of "meta"
#                 after the server name in bb-hosts
#               moved comment explaining purpose of script to start of code
#               caused output of metadb, metastat, and metahs to be displayed
#                 on the web page
# version 2.1 - Warns if default values are not used
#               Applied changes by Todd Jimenez
#               - created get_header and get_footer functions
#               - set summary value
#               - added &red to red alerts
#               - added &yellow to yellow alerts
#               restored functionality to check for the existence of "meta"
#                 after the server name in bb-hosts, but now optional
# version 2.2 - Displays Error messages at the top of page, in case 
# 		the BB server truncates the display message
# version 2.3 - Added ZFS check
#
# BIG BROTHER / XXXXXXXXXXXXXXXX status
#
# Written by Galen Johnson
#  on October 25, 2000
#
# Inspired by a module of the perl clone of BB written
# by Charles Hall in August 1998
#
# Based on code found in the DiskSuite manual
#
# 2.0 Updates by Mike Arnold <mike at razorsedge dot org>
#  on September 27, 2001
#
# 2.1 Updates by Mike Arnold <mike at razorsedge dot org>
#  on November 23, 2001
#
# 2.2 Updates by Rich Armstrong <rich at richardarmstrong dot org dot uk>
#  on November 24, 2003
#
# 2.3 Updates by brad on October 15, 2007

########################################
# NOTE
# This has been tested with BB 1.8c
#
# Tested on :
#   Sun 220R, 420R, E250/450, E4500, SunFire 880
########################################

########################################
# INSTALLATION
#  step 1  - update bb-bbexttab to include this meta
#            (older BB versions update EXT section of the bbdef.sh script)
#
#  step 2 - copy lines mentioned to bbsys.local (without the #'s)
#
#  step 3 - if you are using an older version of BB without bb-bbexttab
#             and you don't want this run on every client uncomment
#             CHECK_BB_HOSTS="Y" and add the name of this $TEST to
#             bb-hosts for this client. eg.
#             myserver1.domain.com   # meta
#
#  step 4 - restart Big Brother
#
# NOTE - the TEST variable in the configuration section, this is the name used
#        as the column header.
########################################

##################################
# CONFIGURE IT HERE
##################################
TEST="raid"
BBPROG="$0"; export BBPROG

#
# Start of lines to put in bbsys.local
# NOTE: MDBIN can be either /usr/sbin or /usr/opt/SUNWmd/sbin
#

# MDBIN=/usr/sbin
# METADB=${MDBIN}/metadb
# METAHS=${MDBIN}/metahs
# METASTAT=${MDBIN}/metastat
# export MDBIN METADB METAHS METASTAT

#
# End of lines to put in bbsys.local
#

# ZFS utility
ZPOOL="/usr/sbin/zpool"
CHECKZFS="Y"

# define colours for graphics
# Comment these out if using older BB versions
RED_PIC="&red"
YELLOW_PIC="&yellow"
GREEN_PIC="&green"

# don't scan through bb-hosts every time
# this is here for older BB versions without bb-bbexttab
# uncomment to activate
#CHECK_BB_HOSTS=Y

# for users that get the "... DATA TRUNCATED ..." message at
# the bottom of the meta page. adds the error message to the
# top of the page
# uncomment to activate
#DATA_IS_LONG=Y

##################################
# Start of script
##################################
#BBHOME="/home/bb/bb"; export BBHOME

if test ! "$BBHOME"
then
        echo "template: BBHOME is not set"
        exit 1
fi

if test ! -d "$BBHOME"
then
        echo "template: BBHOME is invalid"
        exit 1
fi

if test ! "$BBTMP"                      # GET DEFINITIONS IF NEEDED
then
         # echo "*** LOADING BBDEF ***"
        . $BBHOME/etc/bbdef.sh          # INCLUDE STANDARD DEFINITIONS
fi

if test ! -f "$ZPOOL"
then
	# no ZFS utilities found - don't check
	CHECKZFS="N"
fi

get_header()
{
  echo ""
  echo "<FONT SIZE=+2><b>$1</b></FONT> ($2)<BR>"
  # If you do not want the header in a bigger font use line below instead
  #echo "<b>$1</b> ($2)"
  # If you want the "Paul Luzzi" look uncomment this section and comment
  # out the above sections:
  #echo "<P><DIV ALIGN=\"CENTER\"><HR>" 
  #echo "<B>============== $1 ==============</B>"
  #echo "<B>--- ($2) ---</B>"
  #echo "<HR></DIV>" 
  #echo "<BLOCKQUOTE>" 
}

get_footer()
{
  echo ""
  # If you want the "Paul Luzzi" look uncomment this section and comment
  # out the above sections:
  #echo "</BLOCKQUOTE>"
}

#####
#####  Get Status proc - used to get all responses
#####
get_status()
{
  #####
  #####  Setup some variables for use later
  #####
  COLOR="green"

  # Check defaults have been set
  if [ "$MDBIN" = "" ]; then
    MDBIN=/usr/sbin
    echo ""
    ERRMESS="$YELLOW_PIC MDBIN command is not defined in etc/bbsys.local - using default: $MDBIN"
    echo $ERRMESS
  fi

  if [ "$METADB" = "" ]; then
    METADB=${MDBIN}/metadb
    echo ""
    ERRMESS="$YELLOW_PIC METADB command is not defined in etc/bbsys.local - using default: $METADB"
    echo $ERRMESS
  fi

  if [ "$METAHS" = "" ]; then
    METAHS=${MDBIN}/metahs
    echo ""
    ERRMESS="$YELLOW_PIC METAHS command is not defined in etc/bbsys.local - using default: $METAHS"
    echo $ERRMESS
  fi

  if [ "$METASTAT" = "" ]; then
    METASTAT=${MDBIN}/metastat
    echo ""
    ERRMESS="$YELLOW_PIC METASTAT command is not defined in etc/bbsys.local - using default: $METASTAT"
    echo $ERRMESS
  fi

  if [ "$CHECKZFS" = "Y" ]; then
    echo ""
    get_header "ZFS Pools" "$ZPOOL list"
    #zphealth=`${ZPOOL} list | ${TAIL} +2 | ${AWK} '{ print $6 }'`
    zphealth=`${ZPOOL} list | ${TAIL} +2 | ${AWK} '{ fl = $6; if (fl != "ONLINE") print $6 }'`
    #if [ "${zphealth}" != "ONLINE" ]; then
    if [ "${zphealth}" ]; then
      COLOR="red"
      echo ""
      ERRMESS="$RED_PIC <B><I>ZFS Pools are not healthy: </I></B>"
      echo $ERRMESS
      echo ""
      ${ZPOOL} list
    else
      echo ""
      ${ZPOOL} list
    fi
    get_footer
    get_header "ZFS Pools" "$ZPOOL status"
    echo ""
    ${ZPOOL} status
  fi

  ###
  ### Check replicas for problems, capital letters in the flags indicate an error.
  ###
  get_header "MetaDatabases" "$METADB -i"
  dbtrouble=`${METADB} | ${TAIL} +2 | ${AWK} '{ fl = substr($0,1,20); if (fl ~ /[A-Z]/) print $0 }'`
  if [ "${dbtrouble}" ]; then
    COLOR="red"
    echo ""
    ERRMESS="$RED_PIC <B><I>Database replicas are not active: </I></B>"
    echo $ERRMESS
    echo ""
    ${METADB} -i
  else
    ${METADB} -i
  fi
  get_footer

  ###
  ### Check the metadevice state, if the state is not Okay, something is up.
  ###
  get_header "Metadevices" "$METASTAT"
  mdtrouble=`${METASTAT} | ${AWK} '/State:/ { if ( $2 != "Okay" ) print $0 }'`
  if [ "${mdtrouble}" ]; then
    mdtrouble2=`${METASTAT} | ${AWK} '/State:/ { if ( $2 != "Okay" && $2 != "Resyncing" ) print $0 }'`
    if [ "${mdtrouble2}" ]; then 
      COLOR="red"
      echo ""
      ERRMESS="$RED_PIC <B><I>Metadevices are not Okay: </I></B>"
      echo $ERRMESS
      echo ""
      ${METASTAT}
    else
      COLOR="yellow"
      echo ""
      ERRMESS="$YELLOW_PIC <B><I>Metadevices are Resyncing: </I></B>"
      echo $ERRMESS
      echo ""
      ${METASTAT}
    fi
  else
    ${METASTAT}
  fi
  get_footer

  ###
  ### Check the hotspares to see if any have been used.
  ###
  get_header "Hot Spares" "$METAHS -i"
  hstrouble=`${METAHS} -i |  ${AWK} ' /blocks/ { if ( $2 != "Available" ) print $0 }'`
  if [ "${hstrouble}" ]; then
    if [ COLOR != "red" ]; then
      COLOR="yellow"
    fi
    echo ""
    ERRMESS="$YELLOW_PIC <B><I>Hot spares in use: </I></B>"
    echo $ERRMESS
    echo ""
    ${METAHS} -i 2>&1
  else
    ${METAHS} -i 2>&1
  fi
  get_footer

  #####
  #####  Make sure to export COLOR so that it gets back to "central"
  #####
  export COLOR

  #####
  #####  Make sure to export ERRMESS so that it gets back to "central"
  #####
  export ERRMESS

#####
#####  End of get_status proc
#####
}

#####
#####  Main body
#####
if [ "$CHECK_BB_HOSTS" = "Y" ]; then
  # convert "," to "." in the hostname
  MACHINE_WITH_DOTS=`echo $MACHINE | $SED 's/,/\./g'`

  $GREP $MACHINE_WITH_DOTS $BBHOSTS | $GREP "$TEST" |
  while read line
  do
    if [ ! -z "$line" ]; then
      get_status > $BBTMP/$MACHINE.$TEST

      # NOW USE THE BB COMMAND TO SEND THE DATA ACROSS
      if [ "$DATA_IS_LONG" = "Y" ]; then
        $BB $BBDISP "status $BBTMP/$MACHINE.$TEST $COLOR `$DATE` $ERRMESS `$CAT $BBTMP/$MACHINE.$TEST` "
      else
        $BB $BBDISP "status $BBTMP/$MACHINE.$TEST $COLOR `$DATE` `$CAT $BBTMP/$MACHINE.$TEST` "
      fi
    fi
  done
else
  get_status > $BBTMP/$MACHINE.$TEST

  # NOW USE THE BB COMMAND TO SEND THE DATA ACROSS
  if [ "$DATA_IS_LONG" = "Y" ]; then
    $BB $BBDISP "status $MACHINE.$TEST $COLOR `$DATE` $ERRMESS `$CAT $BBTMP/$MACHINE.$TEST`
 "
  else
    $BB $BBDISP "status $MACHINE.$TEST $COLOR `$DATE` `$CAT $BBTMP/$MACHINE.$TEST`
 "
  fi
fi

# Clean up our mess
# Checking for existence of each file since the whole test may be optional
#   and may not actually run on every client
#
if [ -f $BBTMP/$MACHINE.$TEST ]; then
  $RM $BBTMP/$MACHINE.$TEST
fi
##############################################
# end of script
##############################################