celery sometimes hangs

[Mads Kiilerich]

On 07/25/2016 03:07 PM, Thomas De Schampheleire wrote:
> Could you perhaps share the code snippet you are using for the check?

#!/bin/bash

#/etc/crontab:
# ## Check celeryd.log for errors
# */5 *   * * *   root    /srv/kallithea/bin/check-rabbit.sh

LOGFILE=/var/log/kallithea/celeryd.log
SCRIPTNAME=`basename $0`
PATTERN='Connection refused'
RECIPIENTS='me at example.com'
CURRENTTIME=`date "+%Y-%m-%d %H:%m"`

# Check rabbit process
if ! pgrep -f 'rabbitmq[-]server' > /dev/null; then
         echo "[${CURRENTTIME}] - $(hostname) -- rabbit-server dead" | mail -s "[${CURRENTTIME}] - rabbit-server process dead" ${RECIPIENTS} -- -f ono at unity3d.com
         exit 0
fi

# Check for stuck celery
sudo rabbitmqctl list_queues -p kallithea name consumers messages \
         | grep "^celery\>" \
         | while read celery consumers messages; do
                 if [ "$consumers" != "1" ]; then
                         echo "[${CURRENTTIME}] - $(hostname) -- no consumers of celery rabbit queue" | mail -s "[${CURRENTTIME}] - no rabbit queue consumers" ${RECIPIENTS} -- -f ono at unity3d.com
                         exit 0
                 fi
                 if [ "$messages" -gt "20" ]; then
                         {
                                 echo "[${CURRENTTIME}] - $(hostname) -- $messages messages in celery rabbit queue"
                                 sudo service celeryd stop
                                 sleep 1
                                 pkill -f '/srv/kallithea/venv/bin/python /srv/kallithea/venv/bin/paster celeryd /srv/kallithea/kallithea.ini'
                                 sleep 10
                                 pkill -9 -f '/srv/kallithea/venv/bin/python /srv/kallithea/venv/bin/paster celeryd /srv/kallithea/kallithea.ini'
                                 sleep 1
                                 sudo service celeryd start
                         } 2>&1 | mail -s "[${CURRENTTIME}] - $messages messages in celery rabbit queue" ${RECIPIENTS} -- -f ono at unity3d.com
                         exit 0
                 fi
         done

true ok


/Mads