monitower

nimble and trivial monitoring suite
Log | Files | Refs

commit f78c5a4bb2bf422c2a4a413dee1b3f85143bb03e
Author: Josuah Demangeon <me@josuah.net>
Date:   Thu, 17 Sep 2020 00:47:23 +0200

simplify now that we keep each kind of check separate

Diffstat:
Abin/check-load | 7+++++++
Abin/check-mysql-slow-queries | 12++++++++++++
Abin/check-ping | 4++++
Abin/check-ssh | 4++++
Abin/check-tcp | 4++++
Abin/check-tls | 8++++++++
Abin/monit-vmstat | 73+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Abin/monitower-run | 27+++++++++++++++++++++++++++
Abin/monitower-show | 73+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Abin/monitower-status | 16++++++++++++++++
10 files changed, 228 insertions(+), 0 deletions(-)

diff --git a/bin/check-load b/bin/check-load @@ -0,0 +1,7 @@ +#!/bin/sh -eu +for x; do export "var_$x"; done + +: ${var_load:=2} + +load=$(uptime | sed 'y/LOAD,/load./; s/.*load[^0-9]*//; s/[^0-9].*//') +exec [ "$load" -gt "$var_load" ] diff --git a/bin/check-mysql-slow-queries b/bin/check-mysql-slow-queries @@ -0,0 +1,12 @@ +#!/bin/sh -eu +for x; do export "var_$x"; done + +: ${var_time:=30} + +q=" + SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST + WHERE Time > $var_time + AND Command != 'Sleep' +" + +exec [ -z "$(exec mysql -Be "$q")" ] diff --git a/bin/check-ping b/bin/check-ping @@ -0,0 +1,4 @@ +#!/bin/sh -eu +for x; do export "var_$x"; done + +exec ping -c 1 -t 1 "$var_ip" diff --git a/bin/check-ssh b/bin/check-ssh @@ -0,0 +1,4 @@ +#!/bin/sh -eu +for x; do export "var_$x"; done + +exec ssh-keyscan -T 1 "$var_ip" diff --git a/bin/check-tcp b/bin/check-tcp @@ -0,0 +1,4 @@ +#!/bin/sh -eu +for x; do export "var_$x"; done + +exec nc -vz "$var_ip" "$var_port" diff --git a/bin/check-tls b/bin/check-tls @@ -0,0 +1,8 @@ +#!/bin/sh -eu +for x; do export "var_$x"; done + +exec openssl s_client + -servername "$var_dom" \ + -connect "$var_dom:$var_port" \ + -verify_return_error -verify_depth 3 \ + </dev/null diff --git a/bin/monit-vmstat b/bin/monit-vmstat @@ -0,0 +1,73 @@ +#!/usr/bin/awk -f + +function out(name, value) +{ + if (value) + print HOST "." name, $value, NOW +} + +BEGIN { + cmd = "hostname" + cmd | getline HOST + close (cmd) + + sub("\\..*", "", HOST) +} + +$1 ~ /-*procs-*/ { + n = 1 + for (i = 1; match(substr($0, i), "[^ ]+ *"); i += RSTART-1 + RLENGTH) { + header_start[n] = i + header_name[n] = substr($0, i) + sub("^[- ]*", "", header_name[n]) + sub("[- ].*", "", header_name[n]) + n++ + } + next +} + +$1 == "r" { + f = 1 + for (i = 1; i in header_start; i++) { + end = ((i + 1) in header_start) ? header_start[i + 1] : 10000 + section = substr($0, header_start[i], end - header_start[i] - 1) + while (sub(" *[^ ]+ *", "", section)) { + key = header_name[i] "." $f + F[key] = f++ + } + } + next +} + +NR > 3 { + cmd = "date +%s" + cmd | getline NOW + close(cmd) + + gsub("k", "000") + gsub("M", "000000") + gsub("G", "000000000") + gsub("T", "000000000000") + gsub("E", "000000000000000") + + print "" + for (i in F) + print i, "=", $F[i] + + next + out("sys.proc.run", "procs.r") + out("sys.proc.back", "procs.b") + out("sys.mem.avail", "memory.avm") + out("sys.mem.free", "memory.fre") + out("sys.page.fault", "page.flt") + out("sys.interrupt", "int") + out("sys.syscall", "sy") + out("sys.ctxswitch", "ctx") + out("sys.cpu.user", "us") + out("sys.cpu.system", "su") + + for (i in F) { + if (i ~ "^disks.[a-z][0-9]") + out("sys.disk." substr(i, length("disks.")), F[i]) + } +} diff --git a/bin/monitower-run b/bin/monitower-run @@ -0,0 +1,27 @@ +#!/bin/sh -eu + +# This part of monitower reads a single log file as input, and for every line, +# split it as key=values format, and trigger the "$check" for "$host" and store it +# as "$host/$name" in the spool directory. +# +# This way, as few as possible is performed every time, and no complex configuration +# parsing and exploitation is made at every run, and this script is inexpensive enough +# to be run from a /etc/crontab. + +check() { set -eu + for x; do local "var_$x"; done + + mkdir -p "$spool/$var_host/$var_name" + "check-$var_check" "$@" + echo "time=$now exit=$? $*" >>$spool/$var_host/$var_name/current +} + +now=$(date +%s) +spool=/var/spool/monitower +IFS=' +' + +sed -r 's,\\\\,\\s,g; s,\\",\\q,g' "$@" \ +| while IFS=' ' read name check line; do + check $(echo $line | xargs -n 1) +done diff --git a/bin/monitower-show b/bin/monitower-show @@ -0,0 +1,73 @@ +#!/usr/bin/awk -f + +function flush_values(ctx) +{ + printf "\033[%d;1m|\033[m", (32 - (ctx["sum"] < ctx["num"])) + delete ctx["sum"] + delete ctx["num"] + ctx["progress"] += STEP +} + +function add_value(time, value, ctx) +{ + if (!(id in progress)) + ctx["progress"] = START + + if (time > ctx["progress"] && ctx["num"] > 0) + flush_values(ctx) + + # catch up with time by filling the gaps with spaces + while (ctx["progress"] < time) { + printf " " + ctx["progress"] += STEP + } + + # accumulate the sum and num of entries now that we are at the tip + ctx["sum"] += (value == "ok") + ctx["num"] += 1 +} + +function var_split(input, vars, + key, fields, i) +{ + split(input, fields, " ") + for (i in fields) { + key = substr(fields[i], 1, index(fields[i], "=") - 1) + vars[key] = substr(fields[i], index(fields[i], "=") + 1) + } +} + +BEGIN { + cmd = "date +%s" + cmd | getline NOW + close(cmd) + NOW = NOW - NOW % STEP + STEP + + STEP = ENVIRON["STEP"] ? ENVIRON["STEP"] : 600 + COUNT = ENVIRON["COUNT"] ? ENVIRON["COUNT"] : 80 + START = NOW - STEP * COUNT + START = START - START % STEP + STEP + + NAME = ARGV[1] + sub("/*$", "", NAME) + sub(".*/", "", NAME) + + printf " %-15s %s |", NAME, START +} + +START <= $1 { + time = $1 + value = $2 + sub("[^ ]+ [^ ]+ ", "", $0) + + var_split($0, vars) + add_value(time, value, ctx) + + split("", var) +} + +END { + for (id in out) + add_value(NOW, "err", ctx) + printf "| %s\n", NOW +} diff --git a/bin/monitower-status b/bin/monitower-status @@ -0,0 +1,16 @@ +#!/bin/sh -eu + +base=$(cd "${0%/*}/.."; pwd) + +for path in /var/spool/monitor/*/current; do + [ -f "$path" ] || continue + + host=${path%/current} + host=${host##*/} + + echo + echo "$host" + + monitor-show "$path" +done +echo