Integrate also nagios-based monitoring here

1 files changed, 196 insertions, 0 deletions

diff --git a/lib/remote/status.sh b/lib/remote/status.sh
new file mode 100644
index 0000000..7bef6c8
--- /dev/null
+++ b/lib/remote/status.sh
@@ -0,0 +1,196 @@
+. setup/remote.sh
+. lib/remote/print.sh
+. lib/remote/report.sh
+. lib/remote/run.sh
+    
+function check_server_traffic {
+    local result="$1"
+    local yellow=${2:-0}
+    local red=${3:-0}
+    
+    local name=("in" "out" "fwd")
+
+    local traf    
+    read -ra traf <<< $result
+    if [ ${#traf[@]} -ne 3 ]; then
+        [ -z "$result" ] && result="No info received"
+        print "$result" "red" $fold 1 
+        return
+    fi
+    
+    local output=""
+    for i in $(seq 0 2); do
+        if ! [[ ${traf[$i]} =~ ^[0-9]+$ ]]; then
+            print "$result" "red" $fold 1
+            return
+        fi
+
+        local val=$(printf "% 4u" $((${traf[$i]} / 1024 / 1024 / 1024)))
+        if [ $red -gt 0 -a $val -gt $red ]; then
+            val=$(decorate "$val GB" "red")
+        elif [ $yellow -gt 0 -a $val -gt $yellow ]; then
+            val=$(decorate "$val GB" "yellow")
+        else
+            val="$val GB"
+        fi
+        
+#        output="$output${output:+, }${name[$i]}: $val"
+        output="$output${output:+, } $val"
+    done
+    
+#    print "$output"
+    print "in/out/fwd: $output"
+}
+
+function check_server_ {
+    local service=$1 && shift
+
+    local hopo
+    IFS=':' read -ra hopo <<< $1 && shift
+    local host=${hopo[0]}
+    local port=${hopo[1]:-22}
+
+    local result=$(ssh $ssh_params -p $port root@$host /opt/scripts/check_server_$service.sh 2>&1 )
+
+    case "$service" in
+      'traffic')
+        check_server_traffic "$result" "$@"
+        ;;
+      *)
+        print "$result"
+    esac        
+
+}
+
+
+function check_server_status {
+    local hopo
+    IFS=':' read -ra hopo <<< $1 && shift
+    local host=${hopo[0]}
+    local port=${hopo[1]:-22}
+    local services="$@"
+
+    local ping_output=$(scripts/ping.pl $host $port $timeout)
+    print_status "$ping_output"
+
+    if [ "$services" == "-" ]; then return; fi
+
+    if [ "$ping_output" == "1" ]; then
+        local output=$(ssh $ssh_params -p $port root@$host /opt/scripts/check_server_status.sh 2>&1)
+
+        local status=1
+        [ -n "$output" ] && status=0 && output="\n$(decorate "$host:$port" "cyan" "u")\n$output\n"
+        print_status $status
+
+        # Check Additional services, error is reported if anything provides information
+        local headers=""
+        if [ -n "$services" ]; then
+            for service in "$services"; do 
+                if [[ "$service" =~ ^vpn/(.*)$ ]]; then
+                    local ip=${BASH_REMATCH[1]}
+                    [ $(hostname) == "styx" ] && check_server_status "$ip" "-"
+                else
+                    local service_status=1
+                    local service_output=$(ssh $ssh_params -p $port root@$host /opt/scripts/check_${service}_status.sh 2>&1 | sed 's/^[[:space:]]*//;/^$/d')
+                    if [ -n "$service_output" ]; then
+                        first_line=$(echo "$service_output" | head -n 1)
+                        if [[ "$first_line" =~ ^([0-9]+)[[:space:]](.*)$ ]]; then
+                            service_status=${BASH_REMATCH[1]}
+                            service_output=$(sed "1 d" <<< "$service_output")
+                            if [ -n "${BASH_REMATCH[2]}" ]; then
+                                service_header=$(sed -r "s/\\$\{color\s+([^}]*)\}/\$(set_color \\1)/g" <<< "${BASH_REMATCH[2]}")       #"
+                                headers="$headers $service_header"
+                            fi
+                        else
+                            service_status=0
+                        fi
+
+                        if [ -n "$service_output" ]; then
+                            [ -z "$output" ] && output="\n$(decorate "$host:$port" "cyan" "u")"
+                            output="${output}\n  $(decorate "$service" "gray" "u")\n$(echo ${service_output} | sed 's/^/  /')"
+                        fi
+                    fi
+                fi
+
+            done
+        fi
+
+        print " ::: ${headers}"
+        #report="$report<section>$output"
+        if [ -n "$output" ]; then
+            flock -x $0 echo "${output}" >&12
+        fi 
+    else
+        print_status "x"
+        for service in "$services"; do 
+            print_status "x"
+        done
+    fi
+}
+
+function check_service {
+    local service="$1" && shift
+    local id="$1" && shift
+    local args="$@"
+    
+    local output=$(service/check_${service}.sh ${id} "$@" 2>&1  | sed 's/^[[:space:]]*//;/^$/d')
+    
+    local info=$(echo "$output" | tail -n 1)
+    
+    local online=0
+    local health=0
+    local header=""
+    if [[ "$info" =~ ^([0-9]+)[[:space:]]+([0-9]+)[[:space:]]+(.*)$ ]]; then
+        online=${BASH_REMATCH[1]}
+        health=${BASH_REMATCH[2]}
+        header=${BASH_REMATCH[3]}
+
+        output=$(sed '$ d' <<< "$output")
+    fi
+    
+    print_status "$online"
+    print_status "$health"
+
+    header=$(sed -r "s/\\$\{color\s+([^}]*)\}/\$(set_color \\1)/g" <<< "$header")                # "
+    print " ::: ${header}"
+
+
+    important=$(grep "^\*" <<< "$output")
+    messages=$(grep -v "^\*" <<< "$output")
+
+    if [ -n "$output" ]; then
+        output="\n$(decorate "$service:$id" "cyan" "u")\n$important\n$(set_color gray)$messages$(reset_color)\n"
+        flock -x $0 echo "${output}" >&12
+    fi
+}
+
+
+function check__ {
+    local args    
+    local title="$1" && shift
+    read -ra args <<< "$1" && shift
+    local host=${args[0]};
+
+    print "$(decorate $title "purple") ::: " | sed -r "s/\\$\{color\s+([^}]*)\}/\$(set_color \\1)/g; "
+    
+    local service
+    for service in "$@"; do
+        service=$(sed -r "s/<([0-9]+)>/\${args[\\1]}/g; s/<([^>]+)>/\${\\1}/g" <<< "$service")                  # "
+        ( eval "$service" )
+    done
+    print_eol
+}
+
+function check_ {
+    # Buffer the output
+    local output=$(check__ "$@")
+    echo -e "$output"
+}
+
+function check {
+    if [ $parallel -gt 0 ]; then
+        check_ "$@" &
+    else
+        check_ "$@"
+    fi
+}