Wellcome to another Sysadmin & DBA tips, in this post I’ll explain how integrate monit with gtalk, using this system you can control your server services with this Google tool.

Installation

Monit

Monit is a free open source utility for managing and monitoring, processes, programs, files, directories and filesystems on a UNIX system. Monit conducts automatic maintenance and repair and can execute meaningful causal actions in error situations.

The first, install the software:

# aptitude install monit

Second one, configure monit, to do that edit /etc/monit/monitrc. 

This is my monit configuration:

  set daemon 120            # check services at 2-minute intervals
  set logfile /var/log/monit.log
  set idfile /var/lib/monit/id
  set statefile /var/lib/monit/state
  set mailserver localhost
  set mail-format { from: monit@localhost }
  set alert monitor@localhost # Email used to pool with sendxmpp
  set eventqueue
      basedir /var/lib/monit/events 
      slots 100                     

set mail-format {
    subject: monit alert --  $EVENT $SERVICE
    message: $EVENT Service $SERVICE
    Received:
                 Date:        $DATE
                 Action:      $ACTION
                 Host:        $HOST
                 Description: $DESCRIPTION
}

set httpd port 8181 and # Port used for the WEB access, listen on LOCALHOST
    use address localhost  
    allow 0.0.0.0/0.0.0.0        
    allow admin:monit # Username/password

###############################################################################
## Services
###############################################################################
check system localhost
    if loadavg (1min) > 4 then alert
    if loadavg (5min) > 2 then alert
    if memory usage > 75% then alert
    if swap usage > 25% then alert
    if cpu usage (user) > 70% then alert
    if cpu usage (system) > 30% then alert
    if cpu usage (wait) > 20% then alert

# Check the binary Apache file (MD5 checksum + permission + UID + GUID)
check file apache_bin with path /usr/lib/apache2/mpm-prefork/apache2
    if failed checksum and
       expect the sum 762b235as78dfts8s87sd8f7s5 then unmonitor
    if failed permission 755 then unmonitor
    if failed uid root then unmonitor
    if failed gid root then unmonitor
    group web

# Check apache service (service alive + resource consumption)
check process apache with pidfile /var/run/apache2.pid
    start program = "/etc/init.d/apache2 start" with timeout 20 seconds
    stop program  = "/etc/init.d/apache2 stop"
    if children > 150 then alert
    if children > 250 then restart
    if loadavg(5min) greater than 15 for 8 cycles then stop
    if totalcpu > 50% for 2 cycles then alert
    if totalcpu > 80% for 5 cycles then restart
    if failed host srv1.albertolarripa.com port 80 protocol http
	and request "/index.php"
	then restart
    if 5 restarts within 5 cycles then timeout
    group web

# Check ROOT filesystem space usage
check filesystem rootfs with path /
    if space usage > 80% for 5 times within 15 cycles then alert
    group server

# Check DATA filesystem space usage
check filesystem datafs with path /dev/md2
    if space usage > 80% for 5 times within 15 cycles then alert
    group server

# Check nodejs service (service alive)
check process nodejs with pidfile "/tmp/nodejs.pid"
      start program = "/etc/init.d/nodejs start" with timeout 20 seconds
      stop program = "/etc/init.d/nodejs stop"
      if failed port 3005 protocol HTTP
      request /
      with timeout 10 seconds
      then restart
      group web

# Check mysqld service (depends mysqld binary + init.d startup script)
check process mysqld with pidfile /var/run/mysqld/mysqld.pid
   group database
   start program = "/etc/init.d/mysqld start" with timeout 20 seconds
   stop program = "/etc/init.d/mysqld stop"
   if failed host 127.0.0.1 port 3306 protocol mysql then restart
   if 5 restarts within 5 cycles then timeout
   depends on mysql_bin
   depends on mysql_rc

# Check mysql binary file
check file mysql_bin with path /usr/sbin/mysqld
   group database
   if failed checksum then unmonitor
   if failed permission 755 then unmonitor
   if failed uid root then unmonitor
   if failed gid root then unmonitor

# Check mysqld startup script
check file mysql_rc with path /etc/init.d/mysqld
   group database
   if failed checksum then unmonitor
   if failed permission 744 then unmonitor
   if failed uid root then unmonitor
   if failed gid root then unmonitor

# Check mariadb service (depends mariadb binary + init.d startup script)
check process mariadb with pidfile /var/run/mysqld/mariadb.pid
   group database
   start program = "/etc/init.d/mariadb start" with timeout 20 seconds
   stop program = "/etc/init.d/mariadb stop"
   if failed host 127.0.0.1 port 3307 protocol mysql then restart
   if 5 restarts within 5 cycles then timeout
   depends on mariadb_bin
   depends on mariadb_rc

check file mariadb_bin with path /usr/local/mariadb/bin/mysqld
   group database
   if failed checksum then unmonitor
   if failed permission 755 then unmonitor
   if failed uid root then unmonitor
   if failed gid root then unmonitor

check file mariadb_rc with path /etc/init.d/mariadb
   group database
   if failed checksum then unmonitor
   if failed permission 755 then unmonitor
   if failed uid root then unmonitor
   if failed gid root then unmonitor

# Check mongodb process 
check process mongodb with pidfile /usr/local/data/mongo_data/mongod.lock
   group database
   start program = "/etc/init.d/mongodb start" with timeout 20 seconds
   stop  program = "/etc/init.d/mongodb stop"
   if failed host 127.0.0.1 port 27017 protocol http for 3 times within 5 cycles then restart
   if 5 restarts within 5 cycles then timeout

# Check remote server
check host SRV2 with address srv2.albertolarripa.com
    if failed icmp type echo count 3 with timeout 3 seconds then alert
    if failed port 80 protocol http and request "/index.php" with timeout 5 seconds then alert

Monit official manual

Sendxmpp & Centerim

sendxmpp is a perl-script to send xmpp (jabber), similar to what mail(1) does for mail.

CenterIM is a text-mode multi-protocol instant messaging client for Linux, *BSD, Solaris and other Unices. The program supports most of widely used IM protocols, including AIM, ICQ, IRC, MSN, QQ, XMPP (Jabber), Yahoo. CenterIM is a powerful communication tool that can be used through SSH, without the need for VNC/Remote Desktop. CenterIM started as a fork of CenterICQ.

With this software we’ll can send message to our Google accounts.

Create a new user that will be the responsible to manage the integration between the Google talk and Monit services

# useradd monitor

The $HOME directory must be exist

$ /home/monitor

Now install the sendxmpp & centerim software

# apt-get install sendxmpp centerim

Logon with the monitor user and create a new file with your gmail data account

# su - monitor

$ echo "gmailsrv@gmail.com;talk.google.com mygmailpassword gmail.com" >> ~/.sendxmpprc

$ chmod 700 ~/.sendxmpprc

Test if the sendxmpp software work properly

$ echo 'Hello World!!' | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com

Launch centerim and configure with the same account:

$ centerim

Example configure for a new jab account

jab 
├─ Server : talk.google.com:5223 
├─ Secured : yes 
├─ User Name : gmailsrv@gmail.com
├─ Password : ********* 
├─ Priority : 4 
├─ Request receipts: no
├─ Send OS info: no 
├─ OpenPGP key: none

Configuration

Creates a new set of scripts that we’ll use to interactuate with Google chat and monit service

• gtalkdaemon.sh: This script will be listening for new orders
  

$ vim /usr/local/bin/gtalkdaemon.sh

#!/bin/bash
# GTalk Daemon
rm -f /home/monitor/.centerim/jgmailsrv@gmail.com/history
touch /home/monitor/.centerim/jgmailsrv@gmail.com/history
tail -f /home/monitor/.centerim/jgmailsrv@gmail.com/history | perl -ne 'system("/usr/local/bin/gtalkfilter.sh $_")'

• gtalkfilter.sh: This script provide password authentication to increase the security when processing new orders

$ vim /usr/local/bin/gtalkfilter.sh

#!/bin/bash
if [ $(echo $@ | grep <password> | wc -l) = 1 ];
then
 /usr/local/bin/gtalkexecuter.sh $(echo $@ | cut -f2 -d\:);
 > /home/monitor/.centerim/jgmailsrv@gmail.com/history ;
fi

• gtalkexecuter.sh: This script manage your monit services

$ vim /usr/local/bin/gtalkexecuter.sh

#!/bin/bash
log=/var/log/gtalkexecuter.log
case $1 in
  uptime)
   echo "Server status.." | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com ;
   uptime | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com ;
  ;;
  status)
        sudo /usr/bin/monit status | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com ;
 ;;
  summary)
	sudo /usr/bin/monit summary | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com ;
 ;;
  start_apache)
        sudo /usr/bin/monit start apache; sleep 5; sudo /usr/bin/monit summary | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com ;
 ;;
  stop_apache)
        sudo /usr/bin/monit stop apache; sleep 5; sudo /usr/bin/monit summary | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com ;
 ;;
  start_mysql)
        sudo /usr/bin/monit start mysqld; sleep 5; sudo /usr/bin/monit summary | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com ;
 ;;
  stop_mysql)
        sudo /usr/bin/monit stop mysqld; sleep 5; sudo /usr/bin/monit summary | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com ;
 ;;
  mysql_slave)
        mysql --no-defaults --protocol=tcp -h 127.0.0.1 -P 3306 -u monitor -p -e 'show slave status\G' | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com ;
 ;;
  start_mariadb)
        sudo /usr/bin/monit start mariadb; sleep 5; sudo /usr/bin/monit summary | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com ;
 ;;
  stop_mariadb)
        sudo /usr/bin/monit stop mariadb; sleep 5; sudo /usr/bin/monit summary | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com ;
 ;;
  start_mongodb)
        sudo /usr/bin/monit start mongodb; sleep 5; sudo /usr/bin/monit summary | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com ;
 ;;
  stop_mongodb)
        sudo /usr/bin/monit stop mongodb; sleep 5; sudo /usr/bin/monit summary | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com ;
 ;;
  start_nodejs)
        sudo /usr/bin/monit start nodejs_search; sleep 5; sudo /usr/bin/monit summary | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com ;
 ;;
  stop_nodejs)
        sudo /usr/bin/monit stop nodejs_search; sleep 5; sudo /usr/bin/monit summary | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com ;
 ;;
  start_monit)
        sudo /etc/init.d/monit start | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com ;
 ;;
  stop_monit)
        sudo /etc/init.d/monit stop | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com ;
 ;;
  help)
        echo "uptime|status|start_apache|stop_apache|start_mysql|stop_mysql|mysql_slave|start_mariadb|stop_mariadb|start_mongodb|stop_mongodb|start_monit|stop_monit" | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com ;
 ;;

 * )
 echo Unrecognied option $1 >> $log
 ;;
esac

Set execute permissions

$ chmod u+x /usr/local/bin/gtalkexecuter.sh /usr/local/bin/gtalkfilter.sh /usr/local/bin/gtalkdaemon.sh

With the ROOT account create the gtalkexecuter.sh log file:

# touch /var/log/gtalkexecuter.log
# chown monitor.monitor /var/log/gtalkexecuter.log

The monit user must be able to manage the monit services as root. Add this line to your visudo file to provide these permissions

# visudo

monitor ALL=NOPASSWD: /usr/bin/monit, /etc/init.d/monit

Create a script to start/stop gtalk services integration

# cat /etc/init.d/gtalk

#!/bin/bash

case "$1" in
	'start')
	 #Start gtalk monitoring system
	  su - monitor -c 'screen -d -m -S centerim centerim'
	  su - monitor -c 'sleep 5 && /bin/bash /usr/local/bin/gtalkdaemon.sh 2> /dev/null &'
	;;

	'stop')
	  #Stop gtalk monitoring system
	  killall -9 centerim
	  killall -9 tail
	  PROCS=`ps aux | grep gtalk | grep -v grep | awk '{print$2}'`
	  for i in $PROCS; do
		kill -9 $i
	  done
	;;

	'restart')
	 stop
	 start
	;;

	'*')
	 echo "start|stop|restart"
	;;
esac

# cd /etc/rc2.d
# ln -s ../init.d/gtalk S23gtalk
# ls -ltr
lrwxrwxrwx 1 root root  15 Aug  3 22:20 S23gtalk -> ../init.d/gtalk

Finally edit yor /etc/aliases to configure the monitor user mailing sending to

# vim /etc/aliases
monitor: "|/usr/bin/sendxmpp -f /home/monitor/.sendxmpprc -t -u gmailsrv yourmail@albertolarripa.com"

If you would like to receive an ALERT message if some user logon in your server with a specific account like ROOT, add this line in the $HOME/bashrc file:

# vim /root/bashrc

echo 'ALERT - Root Shell Access (srv1.albertolarripa.com) on:' `date` `who` | sendxmpp -t -u gmailsrv yourmail@albertolarripa.com &> /dev/null

Now check if the infrastructure run properly