#!/bin/bash
# Script for use with Nagios to bounce Tomcat Services when they are critical

PATH=/usr/local/bin:/bin:/usr/bin:/sbin:/usr/sbin

mailTO=""

case $1 in
   OK)
      ;;
   WARNING)
      ;;
   CRITICAL)
      if [ $2 == "HARD" ]; then
         echo "restarting glassfish for Dataverse"
     # get stack dump first
         glassfish_pid=`ps -ef | grep java | grep glassfish | grep -v grep | awk '{print $2}'`
         if [ "x${glassfish_pid}" != "x" ]; then
           /usr/bin/jstack $glassfish_pid > "/tmp/glassfish-jstack.${glassfish_pid}"
     fi
         /etc/init.d/glassfish stop
         sleep 20
         glassfish_pid=`ps -ef | grep java | grep glassfish | grep -v grep | awk '{print $2}'`
         if [ "x${glassfish_pid}" != "x" ]; then
            echo "Dataverse glassfish is not responding to init script; trying kill -9"
            kill -9 $glassfish_pid
            sleep 10
         fi
         /etc/init.d/glassfish start
         echo "" | mail -s "bounced glassfish on `hostname`" $mailTO
      fi
      ;;
   UNKNOWN)
   ;;
esac
exit 0

# Monitor Payara and bounce if down. To PAUSE, touch /tmp/stop, then remove /tmp/stop when done!
*/5 * * * * /usr/local/bin/check_site.sh

#!/bin/bash

SITE="http://localhost:8080/loginpage.xhtml"
#SITE="http://localhost:8080/file.xhtml?persistentId=doi:10.7910/DVN/XXXXXX/YYYYYY"
# (we have experimented over the years using more and less expensive pages for
# the purposes of monitoring; using login page, one of the less resource-consuming
# pages, now)


# timeout value is in seconds:
TIMEOUT=180

# This pauses the check. Touch /tmp/stop to disable, remove /tmp/stop to enable
if [ -f /tmp/stop ];then
    exit 0
fi

# Since the curl timeout + whatever overhead may be more than the interval
# at which the cron job runs, let's prevent these jobs from stacking up:

LOCKFILE=/var/lock/check_website.lock

if [ -f $LOCKFILE ]
then
    echo "Site check taking more than 5 minutes!!" | mailx -s "ATTENTION: DELAYED SITE RESPONSE" xxxxx@yyyyy.harvard.edu
    # this implies that the previous instance of this job (run by cron every 5 min.)
    # is still running.

    # the only excuse for a page call to take this long would be if the payara
    # process were in a stop-the-world phase of garbage collection.
    # GC logging (above, and in the Payara log directory) should tell
    # whether that was the case.
    echo -n "bailing out; check already in progress. " >> /var/log/dataverse-bounce.log
    date +%m/%d/%y" "%H:%M >> /var/log/dataverse-bounce.log
    exit 0
fi

touch $LOCKFILE

STAT=`curl -s -o /dev/null -m ${TIMEOUT} -w "%{http_code}" $SITE`

if [ ${STAT}"x" != "200x" ];then

    APP=`cat /xxxxx/payara6/domain1/config/pid.prev`
    /sbin/service payara stop
    sleep 5

    # and, to make sure it's really dead:
    kill -9 $APP 2>/dev/null
    sleep 5
    /sbin/service payara start

    if [ ${STAT}"x" == "000x" ]
    then
    STAT="TIMEOUT"
    fi

    # We have two application nodes in our prod. setup, so on each of the hosts
    # the <hostname> entry in the next two lines indicates the local hostname
    # of the specific node that is being restarted:

    echo "Restarted Payara on <hostname>" | mailx -s "Bounced Payara <hostname>" xxxxx@yyyyy.harvard.edu
    echo -n "Restarted Payara on <hostname>. (http code: ${STAT}) " >> /var/log/dataverse-bounce.log
    date +%m/%d/%y" "%H:%M >> /var/log/dataverse-bounce.log

    # With Payara 6, there is some appreciable time before the completion of asadmin start-domain
    # and the application becoming active and beginning to respond to requests!
    sleep 60
fi

/bin/rm $LOCKFILE