From 6db35a5230578e296d9f493b28e6330e22569c8f Mon Sep 17 00:00:00 2001 From: "Suren A. Chilingaryan" Date: Wed, 4 Sep 2019 22:06:01 +0200 Subject: Integrate also nagios-based monitoring here --- remote/adei.sh | 10 +-- remote/check_all.sh | 2 +- remote/check_kit.sh | 2 +- remote/darksoft.sh | 2 +- remote/ipeserv.sh | 2 +- remote/kaas.sh | 2 +- remote/lib | 1 + remote/lib/README.txt | 34 -------- remote/lib/parameters.sh | 10 --- remote/lib/print.sh | 168 ---------------------------------------- remote/lib/report.sh | 51 ------------ remote/lib/run.sh | 13 ---- remote/lib/status.sh | 196 ----------------------------------------------- remote/logs.sh | 2 +- remote/netstat.sh | 2 +- remote/news.sh | 2 +- remote/setup | 1 + remote/todo.sh | 2 +- 18 files changed, 16 insertions(+), 486 deletions(-) create mode 120000 remote/lib delete mode 100644 remote/lib/README.txt delete mode 100644 remote/lib/parameters.sh delete mode 100644 remote/lib/print.sh delete mode 100644 remote/lib/report.sh delete mode 100644 remote/lib/run.sh delete mode 100644 remote/lib/status.sh create mode 120000 remote/setup (limited to 'remote') diff --git a/remote/adei.sh b/remote/adei.sh index 0b5c42d..c4ba109 100644 --- a/remote/adei.sh +++ b/remote/adei.sh @@ -1,12 +1,12 @@ #! /bin/bash -[ -z $report ] && source lib/status.sh +[ -z $report ] && source lib/remote/status.sh function check_adei { - check "KaaS" "kaas" "check_service adei adei-katrin.kaas.kit.edu/adei 'db_server=fpd&db_name=katrin_rep&db_group=0'" - check "BESS" "pcebess" "check_service adei pcebessadei.competence-e.kit.edu/adei-battery" - check "IMK" "imkmast" "check_service adei imkmastadei.ka.fzk.de/adei" - check "CRD" "crd" "check_service adei adei.crd.yerphi.am/adei" + check "KaaS" "kaas" "check_service adei 'db_server=fpd&db_name=katrin_rep&db_group=0'" + check "BESS" "pcebess" "check_service adei " + check "IMK" "imkmast" "check_service adei " + check "CRD" "crd" "check_service adei " } diff --git a/remote/check_all.sh b/remote/check_all.sh index da2b83f..a506c82 100755 --- a/remote/check_all.sh +++ b/remote/check_all.sh @@ -1,6 +1,6 @@ #! /bin/bash -. lib/status.sh +. lib/remote/status.sh report="short_report" diff --git a/remote/check_kit.sh b/remote/check_kit.sh index 9bdbeae..b50d5ce 100755 --- a/remote/check_kit.sh +++ b/remote/check_kit.sh @@ -1,6 +1,6 @@ #! /bin/bash -. lib/status.sh +. lib/remote/status.sh run kaas.sh run adei.sh diff --git a/remote/darksoft.sh b/remote/darksoft.sh index f8f6949..09823fa 100644 --- a/remote/darksoft.sh +++ b/remote/darksoft.sh @@ -1,6 +1,6 @@ #! /bin/bash -[ -z $report ] && source lib/status.sh +[ -z $report ] && source lib/remote/status.sh function check_darksoft { check "DarkSoft" "darksoft.org" "check_server_status vpn/192.168.21.1" "check_server_ traffic 20 50" diff --git a/remote/ipeserv.sh b/remote/ipeserv.sh index 9f314a1..a737912 100644 --- a/remote/ipeserv.sh +++ b/remote/ipeserv.sh @@ -1,6 +1,6 @@ #! /bin/bash -[ -z $report ] && source lib/status.sh +[ -z $report ] && source lib/remote/status.sh function check_ipe { check "katrin" "katrin.kit.edu" "check_server_status " diff --git a/remote/kaas.sh b/remote/kaas.sh index 5fc1b4a..37d6a5a 100644 --- a/remote/kaas.sh +++ b/remote/kaas.sh @@ -1,6 +1,6 @@ #! /bin/bash -[ -z $report ] && source lib/status.sh +[ -z $report ] && source lib/remote/status.sh function check_kaas { diff --git a/remote/lib b/remote/lib new file mode 120000 index 0000000..dc598c5 --- /dev/null +++ b/remote/lib @@ -0,0 +1 @@ +../lib \ No newline at end of file diff --git a/remote/lib/README.txt b/remote/lib/README.txt deleted file mode 100644 index c5998f8..0000000 --- a/remote/lib/README.txt +++ /dev/null @@ -1,34 +0,0 @@ -conky format -============ - - Server status - * ping works - * no output from 'check_server_status.sh' and additional listed checks - [*] Checks for additional listed services 'check__status.sh'. - - The optional status is in the beginning of first line (0 - red, 1 - green, 2 - yellow). Otherwise, 'red' if any output (also if first word is non-number on the first string). - - If first line starts from the status, the rest of the line is considered a header and reported on the status line. Everything else goes to textual section bellow. - [headers] - - ... Textual information reported by 'check_server_status.sh' and check_*_status.sh scripts. The format - server:port[:service] text.... - - - Extra info for DarkSoft servers - * VPN to the server is working (checked with the ping). This not included in the scripted version, instead server traffic reported directly here. - # # Traffic on VPN interface from/to - - - Server traffic - in/out/forw. Yellow/Red if above specified threshold. No additional errors are reported here. - - - Services (operated by local scripts in 'services' directory) - * Online - * Healthy - [headers] Information about service operation - [errors] Short information on critical errors are reported here (in conky, we push everything in the text section) - - ... Larger textual information about the problems. - - - Service format is following (status is on the last line and mandatory): - * Critical Lines starting with '*' contain critical errors and highlighted (remote) / reported in the service table (conky) - Text This is normal problem description reported in the textual area - # # [header] Two statuses (0 - error, 1 - ok, 2 - warning) and optional header with information about service operation - \ No newline at end of file diff --git a/remote/lib/parameters.sh b/remote/lib/parameters.sh deleted file mode 100644 index c172415..0000000 --- a/remote/lib/parameters.sh +++ /dev/null @@ -1,10 +0,0 @@ -# Configurable -timeout=2 -fold=120 -palete='blue' -parallel=0 -report="standart_report" - - -# Helpers -ssh_params="-x -q -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" -o ConnectTimeout=$timeout" diff --git a/remote/lib/print.sh b/remote/lib/print.sh deleted file mode 100644 index a0fdcbe..0000000 --- a/remote/lib/print.sh +++ /dev/null @@ -1,168 +0,0 @@ -# underlines 4;30,... blinking 5:30,... -# backgrounds: 40,... - -declare -A colors=( - [black]='0;30' - [red]='0;31' - [green]='0;32' - [orange]='0;33' - [blue]='0;34' - [purple]='0;35' - [cyan]='0;36' - [lightgray]='0;37' - [gray]='1;30' - [lightred]='1;31' - [lightgreen]='1;32' - [yellow]='1;33' - [lightblue]='1;34' - [lightpurple]='1;35' - [lightcyan]='1;36' - [white]='1;37' -) - -background="" -foreground="" - - - - -# Fixing color map -#echo -ne "\e[44m" -#echo -ne '\x1B[K' - -function set_background { - if [ -n "$background" ]; then - echo -ne "$background" - echo -ne '\x1B[K' - fi -} - -function set_color { - local color=${1:-""} - local effect=${2:-""} - - c=${colors[$color]} - - - case "$effect" in - 'u') - c=$(echo $c | sed -r "s/^0/4/ ; s/^1/1;4/") - ;; - esac - - [ -n "$color" ] && echo -ne "\033[${c}m" - set_background -} - -function reset_color { - echo -ne "\033[0m" - - set_background - [ -n "$foreground" ] && set_color "$foreground" -} - -function finish { - echo -ne "\033[0m" - echo -} - -function configure_palete { - case $1 in - blue) - background="\e[1;44m" - foreground="white" - ;; - *) - # Lightgray and Yellow are almost invisible on white - colors[yellow]=${colors[orange]} - colors[white]=${colors[black]} - colors[lightgray]=${colors[gray]} - colors[cyan]=${colors[blue]} - ;; - esac - - set_background - reset_color -} - -function decorate { - local output="$1" - local color=${2:-""} - local effect=${3:-""} - - if [ -n "$color" ]; then - set_color $color $effect - echo -ne "${output}" - reset_color - else - echo -ne "$output" - fi -} - -function eol { - sed 's/$/\x1B[K/' -} - -function print_eol { - echo -e '\x1B[K' -} - - - -function print { - local output="$1" - local color=${2:-""} - local char_limit=${3:-$fold} - local line_limit=${4:-0} - - local cmd="decorate \"${output}\" \"$color\" | fold -s -w $char_limit" - [ $line_limit -gt 0 ] && cmd="$cmd | head -n $line_limit" - - eval "$cmd" | eol -} - - -function print_hline { - local color=${1:-"gray"} - local cols=${COLUMNS:-$(tput cols)} - - decorate $(printf '%*s' $cols '' | tr ' ' -) "$color" - echo | eol -} - -function print_table { - # The problem here is that all escapes for colors are treated as normal visible symbols - sed "s/ ::: /@/g" | column -t -s "@" -o " " | eol -} - -function decorate_table { - print_hline - cat - - print_hline -} - -function print_status { - local status="$1" - - if [[ $status =~ ^.$ ]]; then - case $status in - 0) - decorate "*" "red" - ;; - 1) - decorate "*" "green" - ;; - 2) - decorate "*" "yellow" - ;; - *) - decorate "$status" "red" - ;; - esac - else - decorate "?" "red" - fi - echo -n " " -} - -clear diff --git a/remote/lib/report.sh b/remote/lib/report.sh deleted file mode 100644 index e6766a4..0000000 --- a/remote/lib/report.sh +++ /dev/null @@ -1,51 +0,0 @@ -function simple_report { - local generator="$1" - local filter="${2:-cat}" - { - eval "$generator 12>/dev/fd/12 | $filter" - report=$(cat<&12) - } 12<&1 ) - - case "$service" in - 'traffic') - check_server_traffic "$result" "$@" - ;; - *) - print "$result" - esac - -} - - -function check_server_status { - local hopo - IFS=':' read -ra hopo <<< $1 && shift - local host=${hopo[0]} - local port=${hopo[1]:-22} - local services="$@" - - local ping_output=$(scripts/ping.pl $host $port $timeout) - print_status "$ping_output" - - if [ "$services" == "-" ]; then return; fi - - if [ "$ping_output" == "1" ]; then - local output=$(ssh $ssh_params -p $port root@$host /opt/scripts/check_server_status.sh 2>&1) - - local status=1 - [ -n "$output" ] && status=0 && output="\n$(decorate "$host:$port" "cyan" "u")\n$output\n" - print_status $status - - # Check Additional services, error is reported if anything provides information - local headers="" - if [ -n "$services" ]; then - for service in "$services"; do - if [[ "$service" =~ ^vpn/(.*)$ ]]; then - local ip=${BASH_REMATCH[1]} - [ $(hostname) == "styx" ] && check_server_status "$ip" "-" - else - local service_status=1 - local service_output=$(ssh $ssh_params -p $port root@$host /opt/scripts/check_${service}_status.sh 2>&1 | sed 's/^[[:space:]]*//;/^$/d') - if [ -n "$service_output" ]; then - first_line=$(echo "$service_output" | head -n 1) - if [[ "$first_line" =~ ^([0-9]+)[[:space:]](.*)$ ]]; then - service_status=${BASH_REMATCH[1]} - service_output=$(sed "1 d" <<< "$service_output") - if [ -n "${BASH_REMATCH[2]}" ]; then - service_header=$(sed -r "s/\\$\{color\s+([^}]*)\}/\$(set_color \\1)/g" <<< "${BASH_REMATCH[2]}") #" - headers="$headers $service_header" - fi - else - service_status=0 - fi - - if [ -n "$service_output" ]; then - [ -z "$output" ] && output="\n$(decorate "$host:$port" "cyan" "u")" - output="${output}\n $(decorate "$service" "gray" "u")\n$(echo ${service_output} | sed 's/^/ /')" - fi - fi - fi - - done - fi - - print " ::: ${headers}" - #report="$report
$output" - if [ -n "$output" ]; then - flock -x $0 echo "${output}" >&12 - fi - else - print_status "x" - for service in "$services"; do - print_status "x" - done - fi -} - -function check_service { - local service="$1" && shift - local id="$1" && shift - local args="$@" - - local output=$(service/check_${service}.sh ${id} "$@" 2>&1 | sed 's/^[[:space:]]*//;/^$/d') - - local info=$(echo "$output" | tail -n 1) - - local online=0 - local health=0 - local header="" - if [[ "$info" =~ ^([0-9]+)[[:space:]]+([0-9]+)[[:space:]]+(.*)$ ]]; then - online=${BASH_REMATCH[1]} - health=${BASH_REMATCH[2]} - header=${BASH_REMATCH[3]} - - output=$(sed '$ d' <<< "$output") - fi - - print_status "$online" - print_status "$health" - - header=$(sed -r "s/\\$\{color\s+([^}]*)\}/\$(set_color \\1)/g" <<< "$header") # " - print " ::: ${header}" - - - important=$(grep "^\*" <<< "$output") - messages=$(grep -v "^\*" <<< "$output") - - if [ -n "$output" ]; then - output="\n$(decorate "$service:$id" "cyan" "u")\n$important\n$(set_color gray)$messages$(reset_color)\n" - flock -x $0 echo "${output}" >&12 - fi -} - - -function check__ { - local args - local title="$1" && shift - read -ra args <<< "$1" && shift - local host=${args[0]}; - - print "$(decorate $title "purple") ::: " | sed -r "s/\\$\{color\s+([^}]*)\}/\$(set_color \\1)/g; " - - local service - for service in "$@"; do - service=$(sed -r "s/<([0-9]+)>/\${args[\\1]}/g; s/<([^>]+)>/\${\\1}/g" <<< "$service") # " - ( eval "$service" ) - done - print_eol -} - -function check_ { - # Buffer the output - local output=$(check__ "$@") - echo -e "$output" -} - -function check { - if [ $parallel -gt 0 ]; then - check_ "$@" & - else - check_ "$@" - fi -} diff --git a/remote/logs.sh b/remote/logs.sh index 91257c0..e619e78 100755 --- a/remote/logs.sh +++ b/remote/logs.sh @@ -1,6 +1,6 @@ #! /bin/bash -. lib/status.sh +. lib/remote/status.sh function filter_syslog { log="$1" diff --git a/remote/netstat.sh b/remote/netstat.sh index c6c9561..68ca9db 100755 --- a/remote/netstat.sh +++ b/remote/netstat.sh @@ -1,6 +1,6 @@ #! /bin/bash -. lib/status.sh +. lib/remote/status.sh decorate "Incomming:" purple; echo scripts/netstat-in.sh | grep -v "android-d855cbd1c9f827d" | grep -v "pdv-time-capsule-2.ipe.kit.edu" | column -t diff --git a/remote/news.sh b/remote/news.sh index ea24458..844e2ea 100755 --- a/remote/news.sh +++ b/remote/news.sh @@ -1,6 +1,6 @@ #! /bin/bash -. lib/status.sh +. lib/remote/status.sh decorate "Security:" purple; echo rsstail -n 8 -1 -u https://security.gentoo.org/glsa/feed.rss | cut -d ':' -f 2- | cut -c 2- | fmt -t -w $fold diff --git a/remote/setup b/remote/setup new file mode 120000 index 0000000..f4c6f52 --- /dev/null +++ b/remote/setup @@ -0,0 +1 @@ +../setup/ \ No newline at end of file diff --git a/remote/todo.sh b/remote/todo.sh index 863498a..0a6d80d 100755 --- a/remote/todo.sh +++ b/remote/todo.sh @@ -1,6 +1,6 @@ #! /bin/bash -. lib/status.sh +. lib/remote/status.sh -- cgit v1.2.3