Integrate also nagios-based monitoring here

1 files changed, 224 insertions, 0 deletions

diff --git a/adei/check_adei_source.sh b/adei/check_adei_source.sh
new file mode 100755
index 0000000..dcec5bf
--- /dev/null
+++ b/adei/check_adei_source.sh
@@ -0,0 +1,224 @@
+#!/bin/bash
+
+. $(dirname "$0")/adei.sh
+
+
+adei_init "$@"; code=$?
+adei_process_error $code
+
+filter="${adei_args[0]}"
+
+#echo "$adei_url"
+#echo "$adei_source"
+#echo "$filter"
+#exit
+
+status=1
+n_groups=0
+cache_size=0
+cur=`date -u +%s`
+
+group_mode=0
+#groups=`curl -s -m 3 --proxy "" "$adei_url/services/list.php?target=groups&info=1&setup=$2&db_server=$3&db_name=$db_name" | xmllint --format - | grep "Value" | sed -e "s/^.*db_group=\"\([^\"]*\)\".*$/\\1/" | sed -e "s/ /::space::/"`
+groups=$(adei_value_query "list.php?target=groups&info=1" 12); err=$?
+if [ $err -ge 1 -a $err -le 3  ]; then
+    groups=$(adei_value_query "list.php?target=groups" 5); err=$?
+    adei_process_error $err "$groups" "GroupList"
+
+    echo "Data queries are too slow, check database indexes"
+    group_mode=1
+    status=0
+elif [ $err -gt 0 ]; then
+    adei_process_error $err "$groups" "GroupList"
+fi
+
+if [ $group_mode -eq 0 ]; then
+    groups=`echo "$groups" | sed -e "s/ /::space::/g"`
+else
+    groups=`echo "$groups" | sed -e "s/^.*db_group=\"\([^\"]*\)\".*$/\\1/" | sed -e "s/ /::space::/"`
+fi
+
+for group in $groups; do
+    n_groups=$(($n_groups + 1))
+
+    nodata=0
+    if [ $group_mode -eq 0 ]; then
+	group=`echo "$group" | sed -e "s/::space::/ /g"`
+	gid=`echo "$group" | sed -e "s/^.*db_group=\"\([^\"]*\)\".*$/\\1/"`
+	[ "$gid" == "$group" ] && gid=`echo "$group" | sed -e "s/^.*gid=\"\([^\"]*\)\".*$/\\1/"`
+
+	if [ "$gid" == "$group" ]; then
+	    echo "Invalid group specification received from ADEI server: $group"
+	    status=0
+	    continue
+	fi
+	urlgid=`echo $gid | sed -e "s/ /+/g"`
+	last=$group
+    else
+	urlgid=`echo $group | sed -e "s/::space::/+/"`
+	gid=`echo $group | sed -e "s/::space::/ /"`
+	last=$(adei_simple_query "list.php?target=groups&info=1&db_group=$urlgid" 3)
+	if [  $? -ne 0  ]; then
+	    echo "Queries to $gid are too slow, check database indexes"
+	    status=0
+	    nodata=1
+	else 
+    	    error=${last#ERROR: }
+	    if [ "$error" != "$last" ]; then
+		echo "$gid: $error"
+		status=0
+		nodata=1
+	    fi
+	    last=`echo $last | grep "Value"`
+	fi
+	
+    fi
+
+    if [ $nodata -eq 0 ]; then
+        echo "$last" | grep "last=" &> /dev/null
+	if [ $? -eq 0 ]; then
+    	    ts=`echo $last | sed -e "s/^.*last=\"\([^\"]*\)\".*$/\\1/" | cut -d '.' -f 1`
+    	    if [ -z "$ts" ]; then
+    		nodata=1
+#		echo "$gid: contains no data"
+#		if [ $status -eq 1 ]; then status=2; fi
+    	    fi
+	else 
+	    ts=""
+	    nodata=1
+#	    echo "$gid: contains no data"
+#	    if [ $status -eq 1 ]; then status=2; fi
+	fi
+    else
+	ts=$cur
+    fi
+    
+    
+    if [ $nodata -eq 0 ]; then
+	since=$((($cur - $ts) / 3600))
+	if [ $since -gt 0 ]; then
+	    if [ -n "$filter" ]; then
+		echo "$gid" | grep -P "$filter" &>/dev/null
+		accept=$?
+	    else
+		accept=1
+	    fi
+	
+	    if [ $accept -ne 0 ]; then
+        	last=`date -u -d "@$ts" "+%F %R:%S"`
+		echo "$gid: Last updated on $last"
+		if [ $status -eq 1 ]; then
+		    status=2
+		fi
+#	    	continue
+	    fi
+	fi
+    fi
+
+
+    info=$(adei_value_query "info.php?target=cache&db_group=$urlgid"); err=$?
+    [ $err -eq 0 ] && dbsize=`echo $info | sed -e "s/^.*dbsize=\"\([^\"]*\)\".*$/\\1/"`
+
+    if [ $err -ne 0 ]; then
+	echo "$gid: Cache query has failed"
+	status=0
+    elif [ ${#dbsize} -gt 16 ]; then
+	# Otherwise both cache & data are missing. We think it is tolerable
+	if [ $nodata -eq 0 ]; then 
+	    echo "$gid: Cache is empty"
+	    if [ $status -eq 1 ]; then status=2; fi
+	fi
+	continue
+    else
+	cache_size=$(($cache_size + `echo $info | sed -e "s/^.*dbsize=\"\([^\"]*\)\".*$/\\1/"`))
+
+	last_cache=`echo $info | sed -e "s/^.*last=\"\([^\"]*\)\".*$/\\1/"`
+	if [ "$info" == "$last_cache" ]; then
+	    # There is data in the source, but cache is empty (but we should detect this earlier)
+	    # Otherwise, - no cache, no data (which is OK) or we already reported about troublesome source
+	    if [ $nodata -eq 0 ]; then
+		last_cache=0			
+		echo "$gid: Cache is empty (but non-zero size is reported)"
+		if [ $status -eq 1 ]; then status=2; fi
+	    fi
+	    continue
+	else 
+	    if [ -z "$ts" ]; then
+		echo -n "$gid: contains no data, but cached until "
+	        date -u -d "@$last_cache" "+%F %R:%S"
+		if [ $status -eq 1 ]; then status=2; fi
+		continue
+	    else
+		since=$((($ts - $last_cache) / 3600))
+	    fi
+	fi
+	
+	if [ $since -gt 0 ]; then
+	    real_width=$(adei_xml_query "list.php?target=items&db_group=$urlgid"); err=$?
+
+	    if [ $err -ne 0 ]; then
+		status=0
+		echo "$gid: item list service failed"
+	    else
+		real_width=$((`echo "$real_width" | wc -l` - 3))
+	    fi
+
+	    if [ $real_width -le 0 ]; then
+		echo "$gid: Invalid number of items reported by list sevice ($real_width)"
+		status=0
+		continue
+	    fi
+
+	    width=`echo $info | grep width=`
+	    if [ -z "$width" ]; then 
+		echo "$gid: Invalid cache information from ADEI server, width parameter missing"
+		status=0
+		continue
+	    fi
+	    
+	    width=`echo $info | sed -e "s/^.*width=\"\([^\"]*\)\".*$/\\1/"`
+	    if [ $width -eq 0 ]; then
+		echo "$gid: Invalid cache information from ADEI server, width parameter is zero"
+		status=0
+		continue
+	    fi
+	    
+	    found_reason=0
+
+	    if [ "$real_width" -ne "$width" ]; then
+		found_reason=1
+		status=0
+	        echo "$gid: Channel mistmatch $width (cache) and $real_width (reader)"
+	    fi
+		
+	
+#	    if [ $nodata -eq 0 ]; then 
+#		if [ $found_reason -eq 0 ]; then
+#		    # Using current version, we are not able to handle properly limit & filtering together
+#		    adei_version_check "0.0.10"
+#		    if [ $? -eq 0 ]; then
+#			last_data=$(adei_simple_query "getdata.php?db_group=$urlgid&window=0,1")
+#			if [ $? -ne 0 ]; then
+#			    found_reason=1
+#			    status=0
+#			    echo -n "$gid: No valid data in database, but there is (filtered?) records. Last one from "
+#			    date -u -d "@$ts" "+%F %R:%S"
+#			fi
+#		    fi
+#		fi
+#	    fi
+
+	    if [ $found_reason -eq 0 ]; then
+		status=0
+	        echo "$gid: Last $since hours are not cached"
+	    fi
+	fi
+    fi
+done
+
+if [ $n_groups -eq 0 ]; then
+    status=0
+fi
+
+cache_size=$(($cache_size / 1073741824))
+echo "$status $n_groups groups, $cache_size GB"