#!/bin/sh
#
# rm.sh: script to do Raid Manager checking and report back to BB
#
# version 1.7
#
# BIG BROTHER / XXXXXXXXXXXXXXXX status
#
# David Balkwill Version 0.1
#
# tschmidt@micron.com 21-Mar-2003 - Enhanced to detect more errors.
# tschmidt@micron.com 24-Mar-2003 - Continued improvement.
# tschmidt@micron.com 26-Mar-2003 - Add battery age.
# tschmidt@micron.com 23-Apr-2003 - Released as V1.1 on www.deadcat.net.au
# tschmidt@micron.com 02-Jul-2003 - V1.2 adds warning if formatting lun.
# tschmidt@micron.com 10-Jul-2003 - V1.3 added more warnings.
# tschmidt@micron.com 07-Aug-2003 - V1.4 Improved report format.
# tschmidt@micron.com 23-Sep-2003 - V1.5 Move alerts to top for pagers.
# tschmidt@micron.com 30-Mar-2004 - V1.6 Improve runtime by running most
#	commands only once per module.
# tschmidt@micron.com 01-Jun-2004 - V1.7 Add lad output, change default test
#	name from "rm" to "raid".

# set -x

########################################
# NOTE
# This has been tested with BB 1.8b3, 1.9c and 1.9e
########################################

########################################
# INSTALLATION
# 1) make sure this script is in $BBHOME/ext
# 2) Setup sudo so that BB user can run the Raid Manager utilities
# 3) edit $BBHOME/etc/bbdef.sh to call it
########################################

##################################
# CONFIGURE IT HERE
##################################
SUDO="/usr/local/bin/sudo"
HEALTHCK="$SUDO /usr/sbin/osa/healthck"
DRIVUTIL="$SUDO /usr/sbin/osa/drivutil"
RAIDUTIL="$SUDO /usr/sbin/osa/raidutil"
LAD="$SUDO /usr/sbin/osa/lad"

# Place a line in your /etc/sudoers file like this:
#
#     bb ALL = NOPASSWD: /usr/sbin/osa/healthck, /usr/sbin/osa/drivutil, /usr/sbin/osa/raidutil, /usr/sbin/osa/lad
#
# Use the appropriate username for Big Brother and paths to each of the tools

#
# SCRIPTS IN THE BBHOME/ext DIRECTORY ARE ONLY RUN IF
# LISTED IN THE BBEXT VARIABLE OF $BBHOME/runbb.sh
# THIS IS FOR SECURITY.
#

#
# BBPROG SHOULD JUST CONTAIN THE NAME OF THIS FILE
# USEFUL WHEN YOU GET ENVIRONMENT DUMPS TO LOCATE
# THE OFFENDING SCRIPT...
#
BBPROG=rm.sh; export BBPROG

#
# TEST NAME: THIS WILL BECOME A COLUMN ON THE DISPLAY
# IT SHOULD BE AS SHORT AS POSSIBLE TO SAVE SPACE...
# NOTE YOU CAN ALSO CREATE A HELP FILE FOR YOUR TEST
# WHICH SHOULD BE PUT IN www/help/$TEST.html.  IT WILL
# BE LINKED INTO THE DISPLAY AUTOMATICALLY.
#
TEST="raid"
#TEST="rm"	# If you prefer original column header

##################################
# Start of script
##################################

if test ! "$BBHOME"
then
	echo "${BBPROG}: BBHOME is not set"
	exit 1
fi

if test ! -d "$BBHOME"
then
	echo "${BBPROG}: BBHOME is invalid"
	exit 1
fi

if test ! "$BBTMP"			# GET DEFINITIONS IF NEEDED
then
	# echo "*** LOADING BBDEF ***"
	. $BBHOME/etc/bbdef.sh		# INCLUDE STANDARD DEFINITIONS
fi

#
# Initialization
#
eval=0
date=`${DATE}`
RMTMP=$BBTMP/rmcheck.${$}
$RM -f ${RMTMP}*

#
# Check Raid Manager for problems...
#

$LAD > ${RMTMP}lad
for LUN in `cut -d" " -f1 ${RMTMP}lad`
do
	$DRIVUTIL -l $LUN | $GREP -v '^$' | $GREP -v '^     ' >${RMTMP}+
	MODULENAME=`$HEAD -1 ${RMTMP}+ | $SED "s/ Logical Unit Information for //g"`
	MODULE=`echo $MODULENAME | $SED 's/ /_/g'`
	# Only run next commands once per module
	if [ ! -f ${RMTMP}$MODULE ]; then
		$MV -f ${RMTMP}+ ${RMTMP}~
		echo "" >>${RMTMP}~
		$DRIVUTIL -d $LUN | $GREP -v '^$' | $SED 's/;//g' >>${RMTMP}~
		echo "" >>${RMTMP}~
		$HEALTHCK $MODULENAME >${RMTMP}$MODULE
		$CAT ${RMTMP}~ >> ${RMTMP}$MODULE
		$RM -f ${RMTMP}~
		$RAIDUTIL -c $LUN -B | $EGREP "(Battery age|raidutil)" > ${RMTMP}~
		AGE=`$AWK '/Battery/ {print $(NF-4)}' ${RMTMP}~`
		if [ 0$AGE -lt 720 ]; then
			$AWK -v MODULE=$MODULENAME '{if ($0 ~ "Battery") {print MODULE,$0,"&green"} else {print $0}}' ${RMTMP}~ >> ${RMTMP}$MODULE
		else
			$AWK -v MODULE=$MODULENAME '{if ($0 ~ "Battery") {print MODULE,$0,"&yellow"} else {print $0}}' ${RMTMP}~ >> ${RMTMP}$MODULE
			eval=1
		fi
		$RM -f ${RMTMP}~
		echo "" >>${RMTMP}$MODULE
	else
		$RM -f ${RMTMP}+
	fi
done
echo "Array Controller Devices and LUN Balancing" > ${RMTMP}
echo "" >> ${RMTMP}
echo "Device   Serial" >> ${RMTMP}
$CAT ${RMTMP}lad >> ${RMTMP}
echo "" >> ${RMTMP}
${RM} -f ${RMTMP}lad
$CAT ${RMTMP}?* | $GREP -v 'succeeded!' | $GREP -v 'healthck failed' >> ${RMTMP}
$RM -f ${RMTMP}?*

#
# Check for errors in the output since exit status doesn't report failures
# due to sudo. Also add colored icons next to alerts.
#
YLWMSG=""
REDMSG=""
for YELLOW_ALERT in "[Cc]ache [Bb]attery" "[Rr]econstruct" "[Dd]egraded" "[Ff]ormatting" "[Ll]ocked" "[Rr]eplaced" "[Ww]arning" "[Ii]naccessible" "[Uu]nknown" "[Oo]ff [Ll]ine" "[Ee]rror [Oo]ccurred"; do
	$EGREP "$YELLOW_ALERT" ${RMTMP} > ${RMTMP}.yellow 2>>/dev/null
	if [ $? = 0 ]; then
		eval=1
		$AWK -v ALERT="$YELLOW_ALERT" '{if ($0 ~ ALERT) {print $0,"&yellow"} else {print $0}}' ${RMTMP} > ${RMTMP}~
		$MV -f ${RMTMP}~ ${RMTMP}
		YLWMSG="$YLWMSG
`$AWK '{print \"&yellow\",$0}' ${RMTMP}.yellow`"
	fi
done
for RED_ALERT in "[Ff]ail" "[Dd]ead" "[Uu]nresponsive" "Wrong Replaced"; do
	$EGREP "$RED_ALERT" ${RMTMP} > ${RMTMP}.red 2>/dev/null
	if [ $? = 0 ]; then
		eval=2
		$AWK -v ALERT="$RED_ALERT" '{if ($0 ~ ALERT) {print $0,"&red"} else {print $0}}' ${RMTMP} > ${RMTMP}~
		$MV -f ${RMTMP}~ ${RMTMP}
		REDMSG="$REDMSG
`$AWK '{print \"&red\",$0}' ${RMTMP}.red`"
	fi
done
for GREEN_ALERT in "[Oo]ptimal" "Active" "Passive"; do
	$EGREP "$GREEN_ALERT" ${RMTMP} >/dev/null 2>&1
	if [ $? = 0 ]; then
		$AWK -v ALERT="$GREEN_ALERT" '{if ($0 ~ ALERT) {print $0,"&green"} else {print $0}}' ${RMTMP} > ${RMTMP}~
		$MV -f ${RMTMP}~ ${RMTMP}
	fi
done

#
# If any errors occurred, set the STATUS variable
#
if [ ${eval} -ge 2 ]; then
	STATUS="red ${date} Raid Manager Problem Detected
$REDMSG$YLWMSG

`$CAT ${RMTMP}`"
elif [ ${eval} -eq 1 ]; then
	STATUS="yellow ${date} Raid Manager Problem Detected
$YLWMSG

`$CAT ${RMTMP}`"
else
	STATUS="green ${date} Raid Manager OK
`$CAT ${RMTMP}`"
fi

# now tell bb what is up
$BB $BBDISP "status ${MACHINE}.${TEST} $STATUS"

# cleanup and exit
$RM -f ${RMTMP}*
exit ${eval}

##############################################
# end of script
##############################################