dq-jobs

#!/bin/bash

D=$(cd `dirname $0` && pwd)
QDIR=$D/jobs/queue
ADIR=$D/jobs/active
CDIR=$D/jobs/complete

cmd="$1"

if [ "x$cmd" = "x" ]; then
    cmd="active";
fi

function reapstale {
    for j in `ls -1 $ADIR`; do
	if flock -nx $ADIR/$j/lock true; then
	    echo "Cleaning stale job $j"
	    if [ ! -f $ADIR/$j/finish.txt ]; then
		date > $ADIR/$j/finish.txt
	    fi
	    if [ ! -f $ADIR/$j/cleanup.txt ]; then
		date > $ADIR/$j/cleanup.txt
	    fi
	    mv $ADIR/$j $CDIR/$j
	fi
    done
}

case "x$cmd" in
    "xactive")
	reapstale

	AJOBS=$(ls -1 $ADIR 2>/dev/null | fgrep - | sort -nk1 -t-)
	ACNT=$(echo "$AJOBS" | grep . | wc -l)
	echo "Active jobs: $ACNT"
	(
	    echo "JobID Host GPU Runtime Script Env"
	    echo "----- ---- --- ------- ------ ---"
	    for job in $AJOBS; do
		host=$(head -1 $ADIR/$job/host.txt)
		if [ -f $ADIR/$job/start.txt ]; then
		    secs="$((`date '+%s'` - `date -f $ADIR/$job/start.txt '+%s'`))"
		else
		    secs="unk"
		fi
		if [ -f $ADIR/$job/gpu.txt ]; then
		    gpu=$(head -1 $ADIR/$job/gpu.txt)
		else
		    gpu="unk"
		fi
		if [ -f $ADIR/$job/script.txt ]; then
		    script=$(head -1 $ADIR/$job/script.txt)
		else
		    script="unk"
		fi
		if [ -f $ADIR/$job/env.txt ]; then
		    env=$(cut -f2 -d' ' $ADIR/$job/env.txt | tr '\n' ' ')
		else
		    env=""
		fi
		echo "$job $host $gpu ${secs}s $script $env"
	    done
	) | column -t

	QJOBS=$(ls -1 $QDIR 2>/dev/null | fgrep - | sort -nk1 -t-)
	QCNT=$(echo "$QJOBS" | grep . | wc -l)
	if [ $QCNT -gt 0 ]; then
	    echo "Jobs in queue: $QCNT (run 'dq-jobs queue' to inspect)"
	fi
	;;
    "xqueue")
	QJOBS=$(ls -1 $QDIR 2>/dev/null | fgrep - | sort -nk1 -t-)
	QCNT=$(echo "$QJOBS" | grep . | wc -l)
	echo "Jobs in queue: $QCNT"
	(
	    echo "JobID Wait Script Env"
	    echo "----- ---- ------ ---"
	    for job in $QJOBS; do
		if [ -f $QDIR/$job/queue.txt ]; then
		    secs="$((`date '+%s'` - `date -f $QDIR/$job/queue.txt '+%s'`))"
		else
		    secs="unk"
		fi
		if [ -f $QDIR/$job/script.txt ]; then
		    script=$(head -1 $QDIR/$job/script.txt)
		else
		    script="unk"
		fi
		if [ -f $QDIR/$job/env.txt ]; then
		    env=$(cut -f2 -d' ' $QDIR/$job/env.txt | tr '\n' ' ')
		else
		    env=""
		fi
		echo "$job ${secs}s $script $env"
	    done
	) | column -t

	;;
    "xcomplete")
	CJOBS=$(ls -1 $CDIR 2>/dev/null | fgrep - | sort -nk1 -t-)
	CCNT=$(echo "$CJOBS" | grep . | wc -l)
	echo "Completed jobs: $CCNT"
	for job in $CJOBS; do
	    if [ -f $CDIR/$job/finish.txt -a -f $CDIR/$job/start.txt ]; then
		took_str=" - (took $((`date -f $CDIR/$job/finish.txt '+%s'` - `date -f $CDIR/$job/start.txt '+%s'`))s)"
	    fi
	    echo "$job$took_str"
	done

	;;
    "xsummary")
	reapstale

	QCNT=$(ls -1 $QDIR 2>/dev/null | fgrep - | wc -l)
	ACNT=$(ls -1 $ADIR 2>/dev/null | fgrep - | wc -l)
	CCNT=$(ls -1 $CDIR 2>/dev/null | fgrep - | wc -l)

	echo "Active jobs: $ACNT"
	echo "Jobs in queue: $QCNT"
	echo "Completed jobs: $CCNT"
	;;
    *)
	(
	    QJOBS=$(ls -1 $QDIR 2>/dev/null | grep "$cmd")
	    for job in $QJOBS; do
		if [ -f $QDIR/$job/queue.txt ]; then
		    since_str=" - (since $((`date '+%s'` - `date -f $QDIR/$job/queue.txt`))s)"
		fi
		echo "$job: queued$since_str"
	    done
	    AJOBS=$(ls -1 $ADIR 2>/dev/null | grep "$cmd")
	    for job in $AJOBS; do
		if [ -f $ADIR/$job/start.txt ]; then
		    since_str=" - (since $((`date '+%s'` - `date -f $QDIR/$job/start.txt`))s)"
		fi
		echo "$job: active$since_str"
	    done
	    CJOBS=$(ls -1 $CDIR 2>/dev/null | grep "$cmd")
	    for job in $CJOBS; do
		if [ -f $CDIR/$job/finish.txt -a -f $CDIR/$job/start.txt ]; then
		    took_str=" - (took $((`date -f $CDIR/$job/finish.txt '+%s'` - `date -f $CDIR/$job/start.txt '+%s'`))s)"
		fi
		echo "$job: complete$took_str"
	    done
	) | sort -nk1 -t-
	;;
esac