HP Mini Machine Automatic Telnet Inspection Script v1.0

1. Environment Overview There are six HP hosts in total; their telnet IPs are 1.1.1.{1..6}, and the MP card addresses are 1.1.3.{1..6}. Among them, host 1's username and password are user\passwd, host 2's administrator password is 123456, and the rest of the administrator passwords are "password". The script is divided into four parts: telnet_host.exp for telnetting into the host and performing inspection, telnet_console.exp for telnetting into the host's MP card and viewing the logs, auto_check.sh for managing multi-IP login and processing the logs generated by the above two scripts, and a combination of commands used during inspection.

2. Script Scope

3. Check /var/adm/syslog/syslog.log, /etc/rc.log, /etc/shutdownlog, and filter out the error messages;

4. Filter error messages from #dmesg output;

5. Check logical volumes for errors;

6. Check whether any file system usage exceeds 80%;

7. Check hardware for errors;

8. Check physical memory and virtual memory usage;

9. Check disk I/O;

10. Check CPU usage;

11. Script Code and Explanation

#! /bin/bash
# auto_check.sh
#
HOST=host    #Define variable to hold the host IP
USER=user    #Define variable to hold the username
PASSWD=passwd    #Define variable to hold the password
FLAG=flag    #Define variable to distinguish which check is calling the judge function
[ -f ./host.log ] && rm -rf ./host.log
[ -f ./tmp.log ] && rm -rf ./tmp.log
# Determine whether the previous "grep" found the expected value; if not, the log is free of errors and a corresponding message is returned.
function judge_grep() {
if [ $? -ne 0 ];then
echo " All $FLAG take right place." | tee -a ./host.log
else
echo " $FLAG error! The error log has written to file './host.log', please check."
fi
}
# Determine whether the previous statement executed successfully
function judge_state {
if [ $? -eq 0 ];then
echo " $FLAG OK!"
else
echo " $FLAG error! Please check!"
fi
}
for I in {1..6}
do
HOST="1.1.1.$I"
USER="root"
PASSWD="password"
case $I in
1)
USER="user"
PASSWD="passwd"
;;
2)
PASSWD="123456"
;;
esac
echo "Auto Checking $HOST@$USER:"
echo " Checking, please waiting..."
# Invoke the script telnet_host.exp to automatically telnet into the host and run commands; redirect output to ./tmp.log for processing
# (Note: in the output, remove the line "YOU ARE SUPERUSER" to make it easier to find log errors.)
./telnet_host.exp $HOST $USER $PASSWD "`cat ./VMSTATE`" \
"`cat ./RUNSTATE`" "bdf" "ioscan -fn" \
"`cat ./LOGSTATE`" "vgdisplay -v" \
| grep -v -i "WARNING:  YOU ARE SUPERUSER !!" > ./tmp.log
echo " Check finished. Starting analyse data..."
echo "$HOST@$USER:" >> ./host.log
echo >> ./host.log
FLAG=sar;
# Filter sar -d 2 20; sar -u 2 20; sar -w 2 20;
cat ./tmp.log | grep Average -B 2 >> ./host.log
judge_state
echo >> ./host.log
FLAG=vmstat
# Filter vmstat 2 20;
cat ./tmp.log | grep procs -A 2 >> ./host.log
judge_state
echo >> ./host.log
FLAG=fs
# Filter items where filesystem usage is 80% or higher
cat ./tmp.log | grep dev -C 1 \
| grep -v "[0-7][0-9]% \| [0-9]%" | grep % >> ./host.log
judge_state
echo >> ./host.log
FLAG=log
# Filter items in the log output that contain error keywords
cat ./tmp.log | grep -i -e error -e warning -e \
mistake -e notice -e "time out" -e "timed out" >> ./host.log
judge_grep
echo >> ./host.log
FLAG=hardware
# Filter items in the hardware list that contain error keywords
cat ./tmp.log | grep -i -e processor -e disk -e tape \
-e memory -e fc -e lan | grep -i -e no_hw -e \
error -e unknown >> ./host.log
judge_grep
echo >> ./host.log
FLAG=VGs
# Filter items related to VG errors in the output
cat ./tmp.log | grep -i -e "Volume groups" -A 20 \
-e "LV Name" -C 5 -e "PV Name" -C 5 \
| grep -i Status | grep -i -v available
judge_grep
echo >> ./host.log
FLAG=dmesg
# Re-run the command when filtering dmesg to distinguish it from LOG errors
./telnet_host.exp $HOST $USER $PASSWD "dmesg" \
| grep -v -i "WARNING:  YOU ARE SUPERUSER !!" \
| grep -i -e error -e warning -e mistake \
-e notice -e "time out" -e "timed out" >> ./host.log
judge_grep
echo >> ./host.log
# Filtering the TOP output is still under testing; under normal circumstances it would override other results, so it's commented out for now until a better approach is found
# ./telnet_host.exp $HOST $USER $PASSWD "top -d 10" > ./tmp.log
# cat ./tmp.log | grep -i "Cpu states" -A 20 >> ./host.log
# FLAG=top; judge_state
# echo >> ./host.log
done
# The following is the check of the host MP card event log
for I in {1..6}
do
HOST="1.1.3.$I"
USER="Admin"
PASSWD="Admin"
echo
# Invoke telnet_console.exp and pass three parameters; redirect the result to ./tmp.log for processing
./telnet_console.exp $HOST $USER $PASSWD > ./tmp.log
FLAG="Event log"
# Filter the result for lines containing date keywords
cat ./tmp.log | grep -E -e "`date +"%d %b %Y"`" -e `date +%m/%d/%Y` >> host.log
judge_grep
echo >> ./mp.log
done
unset HOST USER PASSWD DATE
rm -rf ./tmp.log
exit 0
#! /usr/bin/expect
# This script is interpreted by expect; that must be declared at the top. For what expect is,
# please refer to Wikipedia http://zh.wikipedia.org/wiki/Expect
# telnet_host.exp
#
#
set timeout -1
set HOST [lindex $argv 0]
set USER [lindex $argv 1]
set PASSWD [lindex $argv 2]
spawn telnet $HOST
expect login:
send "$USER\r"
expect Password:
send "$PASSWD\r"
# Since host #1 is not an admin user and the relevant scripts are defined in .profile, capture the last line of a successful telnet login
# as a marker, and unify PS1 to "#" for convenience below
expect {
"                                       QUIT...........0" {send "\nPS1=#\n"}
"]#" {send "\nPS1=#\n"}
}
# The first three arguments of this script are the host IP, username, and password; from the fourth argument onward are commands or command combinations to execute
# For unknown reasons, commands queued later sometimes appear too early; pressing Enter multiple times can partially resolve this
for {set I 3} {$I<$argc} {incr I} {
expect "#"
send "[lindex $argv $I]\n"
expect "#"
send "\n"
expect "#"
send "\n"
expect "#"
send "\n"
}
# Capture the marker and exit
expect "#"
send "exit\n"
exit 0
expect eof
#! /usr/bin/expect
# telnet_console.exp
#
set timeout -1
set HOST [lindex $argv 0]
set USER [lindex $argv 1]
set PASSWD [lindex $argv 2]
spawn telnet $HOST
expect "MP login:"
send "Admin\n"
expect "MP password"
send "Admin\n"
# In this script's environment, the HP small machine MP card commands come in two versions; capture the different markers separately
expect "MP>"
send "sl\n"
expect {
"MP:VW>" { send "sel\n" }
"*Quit:" { send "e" }
}
# Use sleep intervals to avoid reacting too fast and causing abnormal display
expect "*>"
send "t"
sleep .5
send "a"
sleep .5
send "3"
sleep .5
send "\n"
sleep .5
send "\n"
sleep .5
send "\n"
sleep .5
send "\n"
sleep .5
send "\n"
expect "*>"
send ""
expect "MP>"
send "x\n"
expect eof
exit 0
# ./VMSTAT content
vmstat | head -2; vmstat 2 20 | tail -1
# ./LOGSTATE content
for I in /var/adm/syslog/syslog.log /etc/rc.log /etc/shutdownlog
do
    echo "$I:"
    cat $I | grep  -E "`date +'%b %d'`.*`date +%Y`"
done
# ./RUNSTATE content
for I in u d w
do
    echo "sar -$I 2 20:"
    sar -$I 1 | grep -e usr -e busy -e swpin
    sar -$I 2 20 | grep -i average | sort -rk 3 | head -1
done