`awk '/PERC/{print $1}' afacli.step1.output`
"
# Loop through all identified controllers
#for controller in `awk '/PERC/{print $1}'afacli.step1.output`; do
for controller in `cat afacli.step1.output | grep PERC | awk '{print $1}'`; do
# Build Step 2 - query a controller
echo "open /readonly=true $controller" > afacli.step2
echo "logfile start afacli.step2.output" >> afacli.step2
echo "container list" >> afacli.step2
echo "disk list" >> afacli.step2
echo "disk show partition" >> afacli.step2
echo "disk show smart" >> afacli.step2
echo "controller details" >> afacli.step2
echo " logfile end" >> afacli.step2
echo "exit" >> afacli.step2
# Run Step 2 - query a controller
$CMD_AFACLI < $BBTMP/afacli.step2
AFACLI_OUT2=`cat afacli.step2.output | egrep -v ^'AFA|COMMAND|File' | grep -v 'logfile'`
CHECK_BATTERY="`cat afacli.step2.output | grep 'Battery State'`"
# Add controller queries to the Report
AACRAID_MSG="$AACRAID_MSG
Raid Stats
$AFACLI_OUT2"
# Build Step 3 - list all disks for this controller
echo "open /readonly=true $controller" > afacli.step3
echo "logfile start afacli.step3.output" >> afacli.step3
echo "disk list" >> afacli.step3
echo " logfile end" >> afacli.step3
echo "exit" >> afacli.step3
# Run Step 3 - list all disks for this controller
$CMD_AFACLI < $BBTMP/afacli.step3
#AFACLI_OUT3=`cat afacli.step3.output | egrep -v ^'AFA|COMMAND|File' | grep -v 'logfile'`
for disk in `cat afacli.step3.output|awk '/Disk/{print $1}'`;
do
# Build Step 4 - query each set of controller disks for defects
echo "$disk" >> $AFACLI_STEP4_FILENAME.tmp.disks
echo "open /readonly=true $controller" > afacli.step4
echo "logfile start afacli.step4.output" >> afacli.step4
echo "disk show defects ($disk)" >> afacli.step4
echo " logfile end" >> afacli.step4
echo "exit" >> afacli.step4
# Run Step 4 - query each set of controller disks for defects
$CMD_AFACLI < $BBTMP/afacli.step4
AFACLI_OUT4="$AFACLI_OUT4 `cat afacli.step4.output | egrep -v ^'AFA|COMMAND|logfile|File'`"
done
done
# Add controller queries to the Report
AACRAID_MSG="$AACRAID_MSG
Show Disk Defects
$AFACLI_OUT4" fi # End Gathering Data # Analyze # I haven't had the chance to fully explore all the possible messages, # and so the analysis is sorely lackiing, but it is my understanding # that '!' signifies trouble so we search for that and report it - for now. # One the messages are more fully understood LED blinking will be added to # inidicate the alerted disk. Example, AFA0> disk blink (0:03:0) 30 CHECK1=`echo $AFACLI_OUT2 | grep '!' | wc -l` CHECK2=`echo $CHECK_BATTERY | sed -e 's/Battery State: Ok//g'` BCHECK2=`echo $CHECK_BATTERY | sed -e 's/Battery State: Not Present//g'` CHECK4=`echo $AFACLI_OUT4 | awk '/GROWN/{gsub(/[^[:digit:]]/,""); print}' | grep -v ^0 | wc -l` # Judge MSG="" if [ "$OUT_AFACLI" -eq 0 ]; then STATUS="UNKNOWN" COLOR=yellow MSG="&$COLOR AACRaid Message: Check if permissions on $AFACLI. May need to sticky bit or sudoize.
" elif [ -z "$BBTMP" ]; then STATUS="UNKNOWN" COLOR=yellow MSG="&$COLOR AACRaid Message: Must define BBTMP or some other scratch space.
" elif [ "$CHECK1" -ne 0 ]; then STATUS="ERROR" COLOR=red MSG="&$COLOR AACRaid Message: Something is wrong!
" elif [ -n "$CHECK2" ]; then if [ -n "$BCHECK2" ]; then STATUS="WARNING" COLOR=yellow MSG="&$COLOR AACRaid Message: $CHECK_BATTERY
" fi elif [ "$CHECK4" -ne 0 ]; then STATUS="OPTIMAL" COLOR=yellow MSG="&$COLOR AACRaid Message: Disk Defects are Growing.
" else STATUS="OPTIMAL" MSG="&green AACRaid Message: Relax. Things look good.
" fi # Send Report GLOBAL_MSG="$GLOBAL_MSG $MSG" AACRAID_MSG="$AACRAID_MSG" } MPTRaid1() { # Record the launching of this Routine ROUTINES="$ROUTINES MPTRaid1" # Gather Date CMD_MPT="/usr/local/bin/sudo $MPTSTATUS" # mpt-status command OUT_MPT=`$CMD_MPT` # mpt-status permssions check CHECK1=`$CMD_MPT | grep -c 'state OPTIMAL'` CHECK2=`$CMD_MPT | grep -c 'state ONLINE'` CHECK3=`$CMD_MPT | grep 'state DEGRADED' | grep -c 'RESYNC_IN_PROGRESS'` # Analyze MSG="" if [ -z "$OUT_MPT" ]; then STATUS="UNKNOWN" COLOR=yellow MSG="&$COLOR MPTRaid1 Message: Check if permissions on $MPTSTATUS. May need to sticky bit or sudoize.
" elif [ $CHECK1 -lt 1 ] && [ $CHECK3 -eq 1 ]; then STATUS="DEGRADED" COLOR=yellow MSG="&$COLOR MPTRaid1 Message: RESYNCING
" elif [ $CHECK1 -lt 1 ]; then STATUS="DEGRADED" COLOR=red MSG="&$COLOR MPTRaid1 Message: Not OPTIMAL.
" elif [ $CHECK2 -ne 2 ]; then STATUS="OFFLINE" COLOR=red MSG="&$COLOR MPTRaid1 Message: Failed 'online' check.
" else STATUS="OPTIMAL" MSG="&green MPTRaid1 Message: Relax. Things look good.
" fi # Report GLOBAL_MSG="$GLOBAL_MSG $MSG" MPTRAID1_MSG=" Mirror Status ($CMD_MPT)
$OUT_MPT
" } SoftwareRaid() { # Record the launching of this Routine ROUTINES="$ROUTINES SoftwareRaid" # Gather Data CMD_MD="cat /proc/mdstat" # Kernel Meta-Disk status CMD_AP="$LSRAID -A -p" # Array-based. Listing of all md devices CMD_DP="$LSRAID -D -p" # Disk-based. Description of all member disks OUT_MD=`$CMD_MD` OUT_AP=`$CMD_AP` OUT_DP=`$CMD_DP` # Analyze CHECK1=`cat /proc/mdstat | awk '/block/{gsub(/[\[\]]/,""); print $NF}' | grep -c -v 'U'` CHECK2=`$LSRAID -A -p | awk '/[::alphanum::]/{ print $NF}' | grep -c -v 'online\|spare\|good'` MSG="" if [ $CHECK1 -ne 0 ]; then STATUS="DEGRADED" COLOR=red MSG="&$COLOR SoftwareRaid Message: Failed all 'U' check.
" elif [ -z "$OUT_AP" ]; then STATUS="UNKNOWN" COLOR=yellow MSG="&$COLOR SoftwareRaid Message: Check if permissions on $LSRAID. May need to sticky bit.
" elif [ `echo $OUT_AP | grep -c 'online'` -ne `echo $OUT_AP | grep -c md` ]; then STATUS="OFFLINE" COLOR=red MSG="&$COLOR SoftwareRaid Message: Failed 'online' check.
" elif [ $CHECK2 -ne 0 ]; then STATUS="SUSPECT" COLOR=yellow MSG="&$COLOR SoftwareRaid Message: Check if all disks are ok.
" elif [ $CHECK1 -eq 0 ]; then STATUS="OPTIMAL" MSG="&green SoftwareRaid Message: Relax. Things look good.
" else STATUS="UNKNOWN" COLOR=yellow MSG="&$COLOR SoftwareRaid Message: Status cannot be analyzed.
" fi # Report GLOBAL_MSG="$GLOBAL_MSG $MSG" SOFTWARERAID_MSG=" Kernel Meta-Data Status ($CMD_MD)
$OUT_MD
Array-based ($CMD_AP)
$OUT_AP
Disk-based ($CMD_DP)
$OUT_DP
" } Megaraid() { # Record the launching of this Routine ROUTINES="$ROUTINES Megaraid" # Gather Data CMD_AAI="$MEGARC -AllAdpInfo -nolog" # Adapter Card/Firmware Type CMD_DI="$MEGARC -ldInfo -a0 -Lall -nolog" # Device Status CMD_FI="$MEGARC -pdFailInfo -chAll -idAll -a0 -nolog" # Failure Information CMD_FSI="$MEGARC -FreeSpaceInfo -a0 -nolog" # Free Array Space Information CMD_LPI="$MEGARC -LogPhysInfo -alladp -nolog" # Show Physical Drive Info # These commands are not meant to be run through BB, but are useful for # manual troubleshooting or other automations so here they are. # $MEGARC -chkonLd -a0 -Lall # Check consistency (takes a VERY long time). # $MEGARC -showRbld -a0 # Show Rebuild State (progress) # $MEGARC -getSpeed -chAll -idAll -a0 # Device Speed OUT_AAI=`$SUDO $CMD_AAI | grep -v 'help\|\[A\|\*\|MEGARC\|By\|Scanning\|Finding' | \ grep -v SATA | grep -v "specifying physical"` #OUT_DI=`yes C | $CMD_DI | grep 'rive\|----\|Raid\|Strip\|Status\|0x'` OUT_DI=`$SUDO $CMD_DI | grep -v 'help\|\[A\|\*\|MEGARC\|By\|Scanning\|Finding' | \ grep -v SATA | grep -v "specifying physical"` OUT_FI=`$SUDO $CMD_FI | grep -v 'help\|\[A\|\*\|MEGARC\|By\|Scanning\|Finding' | \ grep '[::space::]' | grep -v SATA | grep -v "specifying physical"` OUT_FSI=`$SUDO $CMD_FSI | grep [::space::] | \ grep -v '\[A\|help\|\*\|MEGARC\|By\|Scanning\|Finding' | grep -v SATA | \ grep -v "specifying physical"` OUT_LPI=`$SUDO $CMD_LPI | grep -v 'help\|\[A\|\*\|MEGARC\|By\|Scanning\|Finding' | \ grep -v SATA | grep -v "specifying physical" | grep 'rive\|RAID'` # Analyze MSG="" NUM=`echo $OUT_DI | grep -c 'Status:'` if [ `echo $OUT_AAI | grep -c 'Error: No PERC Found'` -ne 0 ]; then STATUS="UNKNOWN" COLOR=yellow MSG="&yellow Megaraid Message: No PERC found. Please remove $MEGARC RPM from this host.
" elif [ `echo $OUT_DI | grep -c 'Failed\|Unable'` -ne 0 ]; then STATUS="UNKNOWN" COLOR=yellow MSG="&$COLOR Megaraid Message: PERC found, but $MEGARC is failing. Check that /dev/megadev0 has 644 permissions.
" elif [ `echo $OUT_DI | grep -c RBLD` != "0" ]; then STATUS="REBUILDING" COLOR=yellow MSG="&$COLOR Megaraid Message: RAID is rebuilding ...
" elif [ `echo $OUT_DI | grep -c OPTIMAL` -ne $NUM ]; then STATUS="DEGRADED" COLOR=red MSG="&$COLOR Megaraid Message: Things are bad.
" elif [ `echo $OUT_DI | grep -c OPTIMAL` -eq $NUM ] && [ $NUM -ne 0 ]; then STATUS="OPTIMAL" MSG="&green Megaraid Message: Relax. Things look good.
" else STATUS="UNKNOWN" COLOR=yellow MSG="&$COLOR Megaraid Message: Status cannot be analyzed.
" fi # Report GLOBAL_MSG="$GLOBAL_MSG $MSG" MEGARAID_MSG=" Adapter Card/Firmware Type ($CMD_AAI) $OUT_AAI
Device Status ($CMD_DI)
$OUT_DI
Failure Info ($CMD_FI)
$OUT_FI
Show Physical Device Info ($CMD_LPI)
$OUT_LPI
Free Array Space Information ($CMD_FSI)
$OUT_FSI
" } MegaCli() { # Record the launching of this Routine ROUTINES="$ROUTINES MegaCli" # Gather Data CMD_AAI="$MEGACLI -AdpAllInfo -aALL -NoLog" # Adapter Card/Firmware Type CMD_DI="$MEGACLI -LDInfo -Lall -aALL -NoLog" # Device Status CMD_LPI="$MEGACLI -PDList -aALL -NoLog" # Show Physical Drive Info OUT_AAI=`$SUDO $CMD_AAI` OUT_DI=`$SUDO $CMD_DI` OUT_LPI=`$SUDO $CMD_LPI` DISPLAY_OUT_LPI=`echo "$OUT_LPI" | egrep '^Slot|Error|^Firmware'` #OUT_AAI=`$SUDO $CMD_AAI | grep -v 'help\|\[A\|\*\|MEGARC\|By\|Scanning\|Finding' | \ # grep -v SATA | grep -v "specifying physical"` #OUT_DI=`$SUDO $CMD_DI | grep -v 'help\|\[A\|\*\|MEGARC\|By\|Scanning\|Finding' | \ # grep -v SATA | grep -v "specifying physical"` #OUT_LPI=`$SUDO $CMD_LPI | grep -v 'help\|\[A\|\*\|MEGARC\|By\|Scanning\|Finding' | \ # grep -v SATA | grep -v "specifying physical" | grep 'rive\|RAID'` # Analyze MSG="" NUM=`echo "$OUT_DI" | grep -c 'State'` if [ `echo $OUT_AAI | grep -c 'Error: No PERC Found'` -ne 0 ]; then STATUS="UNKNOWN" COLOR=yellow MSG="&yellow MegaCli Message: No PERC found. Please remove $MEGACLI RPM from this host.
" elif [ `echo $OUT_DI | grep -c 'Failed\|Unable'` -ne 0 ]; then STATUS="UNKNOWN" COLOR=yellow MSG="&$COLOR MegaCli Message: PERC found, but $MEGACLI is failing.
" elif [ `echo $OUT_DI | grep -c RBLD` != "0" ]; then STATUS="REBUILDING" COLOR=yellow MSG="&$COLOR MegaCli Message: RAID is rebuilding ...
" elif [ `echo "$OUT_DI" | grep -c Optimal` -ne $NUM ]; then STATUS="DEGRADED" COLOR=red MSG="&$COLOR MegaCli Message: Things are bad.
" elif [ `echo "$OUT_DI" | grep -c Optimal` -eq $NUM ] && [ $NUM -ne 0 ]; then STATUS="OPTIMAL" MSG="&green MegaCli Message: Relax. Things look good.
" else STATUS="UNKNOWN" COLOR=yellow MSG="&$COLOR MegaCli Message: Status cannot be analyzed.
" fi # Report GLOBAL_MSG="$GLOBAL_MSG $MSG" MEGACLI_MSG=" Device Status ($CMD_DI)
$OUT_DI
Show Physical Device Info ($CMD_LPI)
$DISPLAY_OUT_LPI
Adapter Card/Firmware Type ($CMD_AAI) $OUT_AAI
" } ##### MAIN #################################################################### ### Setup BB Environment and Definitions if [ -z "$BBHOME" ]; then echo "BBHOME is undefined!"; exit 1 #else # . $BBHOME/etc/bbdef.sh fi ### Analyze ROUTINES="" # The criteria to trigger these routines is subject. You may want to differ. if [ -x "$MEGARC" ]; then Megaraid; fi if [ -x "$MEGACLI" ]; then MegaCli; fi #if [ "`cat /proc/mdstat | grep -c 'raid'`" -ne 0 ]; then SoftwareRaid; fi #if [ "`cat /proc/scsi/scsi | grep -c 'PERCRAID'`" -gt 0 ]; then AACRaid; #elif [ "`cat /proc/scsi/scsi | grep -c 'DELL'`" -gt 0 ]; then AACRaid; #if [ "`cat /proc/mpt/version | grep -c 'MPT'`" -gt 0 ]; then MPTRaid1; fi ### Report OUTPUT="