netcheck.sh
743 Zeilen · 26.7 KB · Shell
| 1 | |
| 2 | # |
| 3 | # netcheck.sh - Langzeit-Monitoring der IPv4-/IPv6-Erreichbarkeit |
| 4 | # |
| 5 | # Misst pro Durchlauf und pro Ziel jeweils fuer IPv4 und IPv6: |
| 6 | # - curl-Timing (DNS, TCP-Connect, TLS-Handshake, TTFB, gesamt, HTTP-Status) |
| 7 | # - mtr-Pfadmessung (Loss und Latenz pro Hop, TCP-Modus auf Port 443) |
| 8 | # |
| 9 | # Schreibt eine CSV-Zeile pro Ziel/Protokoll und legt die vollstaendige |
| 10 | # mtr-Ausgabe als Rohlog ab, wenn eine Messung auffaellig war. |
| 11 | # |
| 12 | # Laeuft auf macOS und Linux. Bewusst bash-3.2-kompatibel (macOS-Bordmittel). |
| 13 | # |
| 14 | # Aufruf: |
| 15 | # ./netcheck.sh once einmaliger Lauf mit Ausgabe auf stdout |
| 16 | # ./netcheck.sh run einmaliger Lauf, still (fuer cron/launchd) |
| 17 | # ./netcheck.sh daemon Endlosschleife im Abstand von INTERVAL |
| 18 | # ./netcheck.sh stop beendet einen laufenden daemon sauber |
| 19 | # ./netcheck.sh summary [YYYY-MM] Auswertung der gesammelten Daten |
| 20 | # ./netcheck.sh selftest prueft Voraussetzungen und Konfiguration |
| 21 | # |
| 22 | # Version 1.2 |
| 23 | # |
| 24 | |
| 25 | set -u |
| 26 | |
| 27 | # PATH explizit setzen: cron und launchd starten mit minimalem Environment, |
| 28 | # Homebrew-Tools (mtr) liegen dort sonst ausserhalb des Suchpfads. |
| 29 | PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH" |
| 30 | export PATH |
| 31 | export LC_ALL=C |
| 32 | |
| 33 | SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" |
| 34 | |
| 35 | # --------------------------------------------------------------------------- |
| 36 | # Konfiguration (per netcheck.conf neben dem Script oder per Environment) |
| 37 | # --------------------------------------------------------------------------- |
| 38 | |
| 39 | # Zu pruefende Ziele, durch Leerzeichen getrennt. Hostnamen ohne Schema. |
| 40 | TARGETS="${TARGETS:-tytik.cloud}" |
| 41 | |
| 42 | # Protokollfamilien: "4 6", "4" oder "6" |
| 43 | FAMILIES="${FAMILIES:-4 6}" |
| 44 | |
| 45 | # Ablageort fuer CSV, Rohlogs und Logfile |
| 46 | OUTDIR="${OUTDIR:-$SCRIPT_DIR/data}" |
| 47 | |
| 48 | # Wartezeit zwischen den Laeufen im daemon-Modus (Sekunden) |
| 49 | INTERVAL="${INTERVAL:-300}" |
| 50 | |
| 51 | # mtr: Anzahl Pings pro Hop im Normalfall bzw. bei Auffaelligkeit |
| 52 | MTR_CYCLES="${MTR_CYCLES:-10}" |
| 53 | MTR_ANOMALY_CYCLES="${MTR_ANOMALY_CYCLES:-30}" |
| 54 | |
| 55 | # mtr-Modus. TCP auf 443 ist am aussagekraeftigsten, weil viele CDNs |
| 56 | # UDP-/ICMP-Traceroute verwerfen. "-b" zeigt Hostname und IP. |
| 57 | MTR_MODE="${MTR_MODE:--T -P 443}" |
| 58 | MTR_EXTRA="${MTR_EXTRA:--b}" |
| 59 | |
| 60 | # curl |
| 61 | CURL_METHOD="${CURL_METHOD:-HEAD}" # HEAD oder GET |
| 62 | CURL_PATH="${CURL_PATH:-/}" |
| 63 | CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-10}" |
| 64 | CURL_MAX_TIME="${CURL_MAX_TIME:-20}" |
| 65 | CURL_UA="${CURL_UA:-netcheck/1.0 (Verbindungsmonitoring)}" |
| 66 | |
| 67 | # Schwellwerte, ab denen ein Lauf als auffaellig gilt und ein Rohlog entsteht |
| 68 | LOSS_WARN="${LOSS_WARN:-5}" # Paketverlust zum Ziel in Prozent |
| 69 | RTT_WARN="${RTT_WARN:-150}" # mittlere Latenz zum Ziel in ms |
| 70 | TOTAL_WARN="${TOTAL_WARN:-3000}" # curl-Gesamtzeit in ms |
| 71 | |
| 72 | # Lokalen Netz-Snapshot (Routen, Router-Advertisements, Interface-Adressen) |
| 73 | # bei Auffaelligkeit mitschreiben. 1 = an, 0 = aus. |
| 74 | SNAPSHOT_ON_ANOMALY="${SNAPSHOT_ON_ANOMALY:-0}" |
| 75 | |
| 76 | # Rohlogs aelter als N Tage loeschen. 0 = nie loeschen. |
| 77 | RAW_RETENTION_DAYS="${RAW_RETENTION_DAYS:-30}" |
| 78 | |
| 79 | # Konfigurationsdatei ueberschreibt die Defaults oben |
| 80 | CONF="${NETCHECK_CONF:-$SCRIPT_DIR/netcheck.conf}" |
| 81 | if [ -f "$CONF" ]; then |
| 82 | # shellcheck disable=SC1090 |
| 83 | . "$CONF" |
| 84 | fi |
| 85 | |
| 86 | # Original-stderr auf FD 9 sichern. Waehrend einer mtr-Messung ist stderr in |
| 87 | # die Rohdatei umgeleitet; Meldungen aus dem Signalhandler wuerden dort landen |
| 88 | # statt im Terminal. Ueber FD 9 erreichen sie immer den Benutzer. |
| 89 | exec 9>&2 |
| 90 | |
| 91 | CSV_DIR="$OUTDIR/csv" |
| 92 | RAW_DIR="$OUTDIR/raw" |
| 93 | LOGFILE="$OUTDIR/netcheck.log" |
| 94 | LOCKDIR="$OUTDIR/.lock" |
| 95 | |
| 96 | CSV_HEADER="ts,epoch,runid,target,family,resolved_ip,curl_ok,curl_exit,http_code,dns_ms,connect_ms,tls_ms,ttfb_ms,total_ms,mtr_ok,mtr_loss_pct,mtr_avg_ms,mtr_worst_ms,mtr_max_hop_loss,mtr_hops,dest_reached,anomaly,reason,rawfile" |
| 97 | |
| 98 | VERBOSE=0 |
| 99 | |
| 100 | # Laufzeitzustand fuer die Signalbehandlung |
| 101 | STOP=0 # 1 = Abbruch angefordert, laufende Messung verwerfen |
| 102 | SIGCOUNT=0 # Anzahl empfangener Abbruchsignale |
| 103 | CHILD_PID="" # PID des gerade laufenden Messprozesses |
| 104 | SLEEP_PID="" # PID des laufenden sleep im daemon-Modus |
| 105 | LOCK_HELD=0 |
| 106 | |
| 107 | # --------------------------------------------------------------------------- |
| 108 | # Hilfsfunktionen |
| 109 | # --------------------------------------------------------------------------- |
| 110 | |
| 111 | log() { |
| 112 | printf '%s %s\n' "$(date '+%Y-%m-%dT%H:%M:%S%z')" "$*" >>"$LOGFILE" |
| 113 | [ "$VERBOSE" -eq 1 ] && printf '%s\n' "$*" >&9 |
| 114 | return 0 |
| 115 | } |
| 116 | |
| 117 | die() { |
| 118 | printf 'netcheck: %s\n' "$*" >&2 |
| 119 | exit 1 |
| 120 | } |
| 121 | |
| 122 | # Portabler Timeout-Wrapper: macOS hat kein timeout(1). |
| 123 | # Aufruf: run_timeout SEKUNDEN BEFEHL [ARGS...] |
| 124 | run_timeout() { |
| 125 | _to_secs="$1" |
| 126 | shift |
| 127 | "$@" & |
| 128 | _to_cmd=$! |
| 129 | CHILD_PID="$_to_cmd" |
| 130 | ( sleep "$_to_secs"; kill -TERM "$_to_cmd" 2>/dev/null ) & |
| 131 | _to_watch=$! |
| 132 | wait "$_to_cmd" 2>/dev/null |
| 133 | _to_rc=$? |
| 134 | CHILD_PID="" |
| 135 | kill -TERM "$_to_watch" 2>/dev/null |
| 136 | wait "$_to_watch" 2>/dev/null |
| 137 | return "$_to_rc" |
| 138 | } |
| 139 | |
| 140 | # Sekunden (curl-Format, z.B. 0,123456 oder 0.123456) in Millisekunden |
| 141 | to_ms() { |
| 142 | printf '%s' "$1" | awk '{ gsub(",", ".", $0); printf "%.1f", $0 * 1000 }' |
| 143 | } |
| 144 | |
| 145 | csv_escape() { |
| 146 | # Kommas und Anfuehrungszeichen aus Freitextfeldern entfernen |
| 147 | printf '%s' "$1" | tr ',"' '; ' |
| 148 | } |
| 149 | |
| 150 | have() { command -v "$1" >/dev/null 2>&1; } |
| 151 | |
| 152 | # --------------------------------------------------------------------------- |
| 153 | # Messungen |
| 154 | # --------------------------------------------------------------------------- |
| 155 | |
| 156 | # curl-Timing. Setzt die globalen Variablen c_ok c_exit c_code c_dns c_con |
| 157 | # c_tls c_ttfb c_total c_ip |
| 158 | measure_curl() { |
| 159 | _target="$1" |
| 160 | _fam="$2" |
| 161 | _method_opt="-I -o /dev/null" |
| 162 | [ "$CURL_METHOD" = "GET" ] && _method_opt="-o /dev/null" |
| 163 | |
| 164 | _fmt='%{http_code} %{time_namelookup} %{time_connect} %{time_appconnect} %{time_starttransfer} %{time_total} %{remote_ip}' |
| 165 | _out="" |
| 166 | # shellcheck disable=SC2086 |
| 167 | _out="$(curl "-$_fam" $_method_opt -sS \ |
| 168 | --connect-timeout "$CURL_CONNECT_TIMEOUT" \ |
| 169 | --max-time "$CURL_MAX_TIME" \ |
| 170 | -A "$CURL_UA" \ |
| 171 | -w "$_fmt" \ |
| 172 | "https://${_target}${CURL_PATH}" 2>/dev/null)" |
| 173 | c_exit=$? |
| 174 | |
| 175 | if [ -n "$_out" ]; then |
| 176 | set -- $_out |
| 177 | c_code="${1:-000}" |
| 178 | c_dns="$(to_ms "${2:-0}")" |
| 179 | c_con="$(to_ms "${3:-0}")" |
| 180 | c_tls="$(to_ms "${4:-0}")" |
| 181 | c_ttfb="$(to_ms "${5:-0}")" |
| 182 | c_total="$(to_ms "${6:-0}")" |
| 183 | c_ip="${7:-}" |
| 184 | else |
| 185 | c_code="000"; c_dns=""; c_con=""; c_tls=""; c_ttfb=""; c_total=""; c_ip="" |
| 186 | fi |
| 187 | |
| 188 | if [ "$c_exit" -eq 0 ] && [ "$c_code" != "000" ]; then |
| 189 | c_ok=1 |
| 190 | else |
| 191 | c_ok=0 |
| 192 | fi |
| 193 | } |
| 194 | |
| 195 | # mtr-Pfadmessung. Setzt m_ok m_loss m_avg m_worst m_maxhoploss m_hops |
| 196 | # m_dest_reached und schreibt die Rohausgabe nach "$1" |
| 197 | measure_mtr() { |
| 198 | _target="$1" |
| 199 | _fam="$2" |
| 200 | _cycles="$3" |
| 201 | _rawfile="$4" |
| 202 | |
| 203 | m_ok=0; m_loss=""; m_avg=""; m_worst=""; m_maxhoploss=""; m_hops="" |
| 204 | m_dest_reached=0 |
| 205 | |
| 206 | have mtr || { log "WARN mtr nicht gefunden - Pfadmessung uebersprungen"; return 1; } |
| 207 | |
| 208 | # mtr braucht Raw Sockets. Als root direkt, sonst passwortloses sudo. |
| 209 | _mtr="mtr" |
| 210 | if [ "$(id -u)" -ne 0 ]; then |
| 211 | if sudo -n true 2>/dev/null; then |
| 212 | _mtr="sudo -n mtr" |
| 213 | fi |
| 214 | fi |
| 215 | |
| 216 | # Grosszuegiger Timeout: ein Zyklus pro Sekunde plus Puffer |
| 217 | _timeout=$(( _cycles + 30 )) |
| 218 | |
| 219 | # shellcheck disable=SC2086 |
| 220 | run_timeout "$_timeout" $_mtr "-$_fam" $MTR_MODE $MTR_EXTRA -r -w \ |
| 221 | -c "$_cycles" "$_target" >"$_rawfile" 2>&1 |
| 222 | _rc=$? |
| 223 | |
| 224 | if [ "$_rc" -ne 0 ] && [ ! -s "$_rawfile" ]; then |
| 225 | log "WARN mtr -$_fam $_target fehlgeschlagen (exit $_rc)" |
| 226 | return 1 |
| 227 | fi |
| 228 | |
| 229 | # Aus dem Report die Hop-Zeilen auswerten. Die letzten sieben Spalten sind |
| 230 | # immer Loss% Snt Last Avg Best Wrst StDev - unabhaengig davon, wie breit |
| 231 | # die Hostspalte ist. Deshalb wird von hinten indiziert. |
| 232 | _parsed="$(awk ' |
| 233 | BEGIN { maxloss = 0; hops = 0; last_loss = 0; last_avg = 0; last_wrst = 0; last_host = "???" } |
| 234 | /\|--/ { |
| 235 | loss = $(NF-6); sub("%", "", loss); |
| 236 | avg = $(NF-3); |
| 237 | wrst = $(NF-1); |
| 238 | host = $2; |
| 239 | hops++; |
| 240 | if (loss+0 > maxloss+0) maxloss = loss; |
| 241 | last_loss = loss; last_avg = avg; last_wrst = wrst; last_host = host; |
| 242 | } |
| 243 | END { |
| 244 | if (hops == 0) { print "NA"; exit } |
| 245 | reached = (last_host == "???" || last_loss+0 >= 100) ? 0 : 1; |
| 246 | printf "%s %s %s %s %d %d\n", last_loss, last_avg, last_wrst, maxloss, hops, reached; |
| 247 | }' "$_rawfile")" |
| 248 | |
| 249 | [ "$_parsed" = "NA" ] && { log "WARN mtr -$_fam $_target lieferte keine Hops"; return 1; } |
| 250 | |
| 251 | set -- $_parsed |
| 252 | m_loss="${1:-}"; m_avg="${2:-}"; m_worst="${3:-}"; m_maxhoploss="${4:-0}" |
| 253 | m_hops="${5:-0}"; m_dest_reached="${6:-0}" |
| 254 | m_ok=1 |
| 255 | return 0 |
| 256 | } |
| 257 | |
| 258 | # Lokaler Netz-Snapshot bei Auffaelligkeit |
| 259 | snapshot() { |
| 260 | _file="$1" |
| 261 | { |
| 262 | echo "=== $(date '+%Y-%m-%dT%H:%M:%S%z') lokaler Netz-Snapshot ===" |
| 263 | if [ "$(uname -s)" = "Darwin" ]; then |
| 264 | echo "--- scutil --nwi"; scutil --nwi 2>&1 |
| 265 | echo "--- netstat -rn -f inet6 | grep default" |
| 266 | netstat -rn -f inet6 2>/dev/null | grep default |
| 267 | echo "--- netstat -rn -f inet | grep default" |
| 268 | netstat -rn -f inet 2>/dev/null | grep default |
| 269 | echo "--- ndp -rn"; ndp -rn 2>&1 |
| 270 | echo "--- ifconfig (global)"; ifconfig 2>/dev/null | grep -E '^[a-z]|inet ' |
| 271 | elif have ip; then |
| 272 | echo "--- ip -6 route"; ip -6 route 2>&1 |
| 273 | echo "--- ip -4 route"; ip -4 route 2>&1 |
| 274 | echo "--- ip -brief addr"; ip -brief addr 2>&1 |
| 275 | echo "--- ip -6 neigh"; ip -6 neigh 2>&1 |
| 276 | else |
| 277 | echo "--- netstat -rn (iproute2 nicht vorhanden)" |
| 278 | netstat -rn 2>&1 |
| 279 | echo "--- ifconfig"; ifconfig 2>&1 | grep -E '^[a-z]|inet' |
| 280 | fi |
| 281 | } >>"$_file" 2>&1 |
| 282 | } |
| 283 | |
| 284 | # --------------------------------------------------------------------------- |
| 285 | # Ein Durchlauf |
| 286 | # --------------------------------------------------------------------------- |
| 287 | |
| 288 | do_run() { |
| 289 | _ts="$(date '+%Y-%m-%dT%H:%M:%S%z')" |
| 290 | _epoch="$(date '+%s')" |
| 291 | _runid="$(date '+%Y%m%d-%H%M%S')" |
| 292 | _day="$(date '+%Y-%m-%d')" |
| 293 | _month="$(date '+%Y-%m')" |
| 294 | _csv="$CSV_DIR/netcheck-$_month.csv" |
| 295 | |
| 296 | [ -f "$_csv" ] || printf '%s\n' "$CSV_HEADER" >"$_csv" |
| 297 | mkdir -p "$RAW_DIR/$_day" |
| 298 | |
| 299 | for _target in $TARGETS; do |
| 300 | for _fam in $FAMILIES; do |
| 301 | |
| 302 | [ "$STOP" -eq 1 ] && break |
| 303 | |
| 304 | measure_curl "$_target" "$_fam" |
| 305 | |
| 306 | # Wurde die Messung durch ein Abbruchsignal zerrissen, waere das |
| 307 | # Ergebnis ein Scheinausfall. Solche Werte gehoeren nicht in die |
| 308 | # Auswertung, deshalb wird die Messung ersatzlos verworfen. |
| 309 | if [ "$STOP" -eq 1 ]; then |
| 310 | log "SKIP $_target IPv$_fam waehrend Abbruch verworfen (nicht gewertet)" |
| 311 | break |
| 312 | fi |
| 313 | |
| 314 | _tmpraw="$RAW_DIR/$_day/.tmp-$_runid-$_target-v$_fam.txt" |
| 315 | measure_mtr "$_target" "$_fam" "$MTR_CYCLES" "$_tmpraw" |
| 316 | |
| 317 | if [ "$STOP" -eq 1 ]; then |
| 318 | rm -f "$_tmpraw" |
| 319 | log "SKIP $_target IPv$_fam waehrend Abbruch verworfen (nicht gewertet)" |
| 320 | break |
| 321 | fi |
| 322 | |
| 323 | # Auffaelligkeit bewerten |
| 324 | _anomaly=0 |
| 325 | _reason="" |
| 326 | |
| 327 | if [ "$c_ok" -ne 1 ]; then |
| 328 | _anomaly=1 |
| 329 | _reason="curl fehlgeschlagen (exit $c_exit, http $c_code)" |
| 330 | elif [ -n "$c_total" ] && [ "$(awk -v a="$c_total" -v b="$TOTAL_WARN" 'BEGIN{print (a>b)?1:0}')" = "1" ]; then |
| 331 | _anomaly=1 |
| 332 | _reason="curl langsam (${c_total} ms)" |
| 333 | fi |
| 334 | |
| 335 | case "$c_code" in |
| 336 | ""|000|5*) _anomaly=1; [ -z "$_reason" ] && _reason="HTTP-Status $c_code" ;; |
| 337 | esac |
| 338 | |
| 339 | if [ "$m_ok" -eq 1 ]; then |
| 340 | if [ "$m_dest_reached" -ne 1 ]; then |
| 341 | _anomaly=1 |
| 342 | _reason="${_reason:+$_reason; }Ziel im mtr nicht erreicht" |
| 343 | fi |
| 344 | if [ "$(awk -v a="$m_loss" -v b="$LOSS_WARN" 'BEGIN{print (a+0>b+0)?1:0}')" = "1" ]; then |
| 345 | _anomaly=1 |
| 346 | _reason="${_reason:+$_reason; }Paketverlust ${m_loss}%" |
| 347 | fi |
| 348 | if [ "$(awk -v a="$m_avg" -v b="$RTT_WARN" 'BEGIN{print (a+0>b+0)?1:0}')" = "1" ]; then |
| 349 | _anomaly=1 |
| 350 | _reason="${_reason:+$_reason; }Latenz ${m_avg} ms" |
| 351 | fi |
| 352 | elif have mtr; then |
| 353 | _anomaly=1 |
| 354 | _reason="${_reason:+$_reason; }mtr ohne Ergebnis" |
| 355 | fi |
| 356 | |
| 357 | # Rohlog nur bei Auffaelligkeit behalten, dann aber mit mehr Zyklen |
| 358 | _rawname="" |
| 359 | if [ "$_anomaly" -eq 1 ]; then |
| 360 | _rawname="$_day/$_runid-$_target-v$_fam.txt" |
| 361 | _rawpath="$RAW_DIR/$_rawname" |
| 362 | { |
| 363 | echo "=== $_ts $_target IPv$_fam ===" |
| 364 | echo "Grund: $_reason" |
| 365 | echo "curl: exit=$c_exit http=$c_code ip=$c_ip dns=${c_dns}ms connect=${c_con}ms tls=${c_tls}ms ttfb=${c_ttfb}ms total=${c_total}ms" |
| 366 | echo |
| 367 | echo "--- mtr Kurzlauf ($MTR_CYCLES Zyklen) ---" |
| 368 | } >"$_rawpath" |
| 369 | [ -f "$_tmpraw" ] && cat "$_tmpraw" >>"$_rawpath" |
| 370 | |
| 371 | if have mtr && [ "$MTR_ANOMALY_CYCLES" -gt "$MTR_CYCLES" ]; then |
| 372 | { |
| 373 | echo |
| 374 | echo "--- mtr Nachmessung ($MTR_ANOMALY_CYCLES Zyklen) ---" |
| 375 | } >>"$_rawpath" |
| 376 | _tmpraw2="$_tmpraw.2" |
| 377 | measure_mtr "$_target" "$_fam" "$MTR_ANOMALY_CYCLES" "$_tmpraw2" || true |
| 378 | [ -f "$_tmpraw2" ] && cat "$_tmpraw2" >>"$_rawpath" && rm -f "$_tmpraw2" |
| 379 | if [ "$STOP" -eq 1 ]; then |
| 380 | echo "(Nachmessung durch Abbruchsignal beendet - unvollstaendig)" >>"$_rawpath" |
| 381 | fi |
| 382 | fi |
| 383 | |
| 384 | [ "$SNAPSHOT_ON_ANOMALY" = "1" ] && snapshot "$_rawpath" |
| 385 | |
| 386 | log "ANOMALY $_target IPv$_fam: $_reason -> raw/$_rawname" |
| 387 | fi |
| 388 | rm -f "$_tmpraw" |
| 389 | |
| 390 | printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' \ |
| 391 | "$_ts" "$_epoch" "$_runid" "$_target" "$_fam" "$(csv_escape "$c_ip")" \ |
| 392 | "$c_ok" "$c_exit" "$c_code" \ |
| 393 | "$c_dns" "$c_con" "$c_tls" "$c_ttfb" "$c_total" \ |
| 394 | "$m_ok" "$m_loss" "$m_avg" "$m_worst" "$m_maxhoploss" "$m_hops" \ |
| 395 | "$m_dest_reached" "$_anomaly" "$(csv_escape "$_reason")" "$_rawname" \ |
| 396 | >>"$_csv" 2>/dev/null |
| 397 | |
| 398 | # Schreibfehler nicht verschlucken - eine stumm verlorene Messung |
| 399 | # ist schlimmer als eine laute Fehlermeldung. |
| 400 | if [ ! -w "$_csv" ]; then |
| 401 | log "FEHLER CSV nicht beschreibbar: $_csv" |
| 402 | printf 'netcheck: FEHLER - kann nicht nach %s schreiben.\n' "$_csv" >&9 |
| 403 | fi |
| 404 | |
| 405 | if [ "$VERBOSE" -eq 1 ]; then |
| 406 | if [ "$_anomaly" -eq 1 ]; then |
| 407 | printf ' %-24s IPv%s AUFFAELLIG %s\n' "$_target" "$_fam" "$_reason" |
| 408 | else |
| 409 | printf ' %-24s IPv%s ok http=%s ttfb=%sms loss=%s%% avg=%sms hops=%s\n' \ |
| 410 | "$_target" "$_fam" "$c_code" "$c_ttfb" "${m_loss:-n/a}" "${m_avg:-n/a}" "${m_hops:-n/a}" |
| 411 | fi |
| 412 | fi |
| 413 | done |
| 414 | [ "$STOP" -eq 1 ] && break |
| 415 | done |
| 416 | |
| 417 | if [ "$VERBOSE" -eq 1 ]; then |
| 418 | _rows=$(( $(wc -l <"$_csv" 2>/dev/null || echo 1) - 1 )) |
| 419 | printf ' -> %s Zeilen in %s\n' "$_rows" "$_csv" |
| 420 | fi |
| 421 | |
| 422 | # Alte Rohlogs aufraeumen |
| 423 | if [ "$RAW_RETENTION_DAYS" -gt 0 ] && [ -d "$RAW_DIR" ]; then |
| 424 | find "$RAW_DIR" -type f -name '*.txt' -mtime "+$RAW_RETENTION_DAYS" -delete 2>/dev/null |
| 425 | find "$RAW_DIR" -mindepth 1 -type d -empty -delete 2>/dev/null |
| 426 | fi |
| 427 | } |
| 428 | |
| 429 | # --------------------------------------------------------------------------- |
| 430 | # Auswertung |
| 431 | # --------------------------------------------------------------------------- |
| 432 | |
| 433 | do_summary() { |
| 434 | _month="${1:-}" |
| 435 | if [ -n "$_month" ]; then |
| 436 | _files="$CSV_DIR/netcheck-$_month.csv" |
| 437 | else |
| 438 | _files="$CSV_DIR"/netcheck-*.csv |
| 439 | fi |
| 440 | |
| 441 | # shellcheck disable=SC2086 |
| 442 | ls $_files >/dev/null 2>&1 || die "keine CSV-Daten in $CSV_DIR gefunden" |
| 443 | |
| 444 | # shellcheck disable=SC2086 |
| 445 | awk -F, ' |
| 446 | FNR == 1 { next } |
| 447 | NF < 20 { next } |
| 448 | { |
| 449 | ts = $1; target = $4; fam = $5; |
| 450 | curl_ok = $7; total = $14; |
| 451 | mtr_ok = $15; loss = $16; avg = $17; |
| 452 | anomaly = $22; reason = $23; |
| 453 | |
| 454 | key = target " IPv" fam; |
| 455 | if (!(key in seen)) { seen[key] = 1; order[++n] = key } |
| 456 | |
| 457 | runs[key]++; |
| 458 | if (curl_ok != 1) fails[key]++; |
| 459 | if (anomaly == 1) anos[key]++; |
| 460 | |
| 461 | if (curl_ok == 1 && total != "") { |
| 462 | sum_total[key] += total; cnt_total[key]++; |
| 463 | if (total + 0 > max_total[key] + 0) max_total[key] = total; |
| 464 | } |
| 465 | if (mtr_ok == 1 && avg != "") { |
| 466 | sum_avg[key] += avg; cnt_avg[key]++; |
| 467 | if (avg + 0 > max_avg[key] + 0) max_avg[key] = avg; |
| 468 | sum_loss[key] += loss; cnt_loss[key]++; |
| 469 | } |
| 470 | |
| 471 | if (anomaly == 1) { |
| 472 | hour = substr(ts, 1, 13); |
| 473 | hits[hour]++; |
| 474 | day = substr(ts, 1, 10); |
| 475 | dayhits[day]++; |
| 476 | if (!(reason in rcount)) rorder[++rn] = reason; |
| 477 | rcount[reason]++; |
| 478 | if (first[key] == "") first[key] = ts; |
| 479 | last[key] = ts; |
| 480 | } |
| 481 | if (window_first == "") window_first = ts; |
| 482 | window_last = ts; |
| 483 | total_runs++; |
| 484 | } |
| 485 | END { |
| 486 | if (total_runs == 0) { print "Keine Datenzeilen gefunden."; exit } |
| 487 | printf "Messzeitraum : %s bis %s\n", window_first, window_last; |
| 488 | printf "Messungen : %d\n\n", total_runs; |
| 489 | |
| 490 | printf "%-28s %7s %7s %8s %10s %10s %9s %9s\n", \ |
| 491 | "Ziel", "Laeufe", "Fehler", "Auffaell", "Ø total ms", "max total", "Ø loss %", "Ø rtt ms"; |
| 492 | for (i = 1; i <= n; i++) { |
| 493 | k = order[i]; |
| 494 | at = (cnt_total[k] ? sum_total[k] / cnt_total[k] : 0); |
| 495 | al = (cnt_loss[k] ? sum_loss[k] / cnt_loss[k] : 0); |
| 496 | ar = (cnt_avg[k] ? sum_avg[k] / cnt_avg[k] : 0); |
| 497 | printf "%-28s %7d %7d %8d %10.1f %10.1f %9.2f %9.1f\n", \ |
| 498 | k, runs[k], fails[k]+0, anos[k]+0, at, max_total[k]+0, al, ar; |
| 499 | } |
| 500 | |
| 501 | if (rn > 0) { |
| 502 | printf "\nHaeufigste Ursachen\n"; |
| 503 | for (i = 1; i <= rn; i++) |
| 504 | printf " %5d x %s\n", rcount[rorder[i]], rorder[i]; |
| 505 | |
| 506 | printf "\nAuffaellige Laeufe pro Tag\n"; |
| 507 | for (d in dayhits) daylist[++dn] = d; |
| 508 | for (i = 1; i < dn; i++) for (j = i+1; j <= dn; j++) |
| 509 | if (daylist[i] > daylist[j]) { t = daylist[i]; daylist[i] = daylist[j]; daylist[j] = t } |
| 510 | for (i = 1; i <= dn; i++) { |
| 511 | bar = ""; c = dayhits[daylist[i]]; |
| 512 | for (b = 0; b < c && b < 60; b++) bar = bar "#"; |
| 513 | printf " %s %4d %s\n", daylist[i], c, bar; |
| 514 | } |
| 515 | |
| 516 | printf "\nAuffaellige Laeufe pro Stunde (Tagesverteilung)\n"; |
| 517 | for (h in hits) { hh = substr(h, 12, 2); hourly[hh] += hits[h] } |
| 518 | for (i = 0; i < 24; i++) { |
| 519 | hh = sprintf("%02d", i); |
| 520 | c = hourly[hh] + 0; |
| 521 | bar = ""; for (b = 0; b < c && b < 60; b++) bar = bar "#"; |
| 522 | printf " %s Uhr %4d %s\n", hh, c, bar; |
| 523 | } |
| 524 | } else { |
| 525 | printf "\nKeine Auffaelligkeiten im Zeitraum.\n"; |
| 526 | } |
| 527 | }' $_files |
| 528 | } |
| 529 | |
| 530 | # --------------------------------------------------------------------------- |
| 531 | # Selbsttest |
| 532 | # --------------------------------------------------------------------------- |
| 533 | |
| 534 | do_selftest() { |
| 535 | echo "netcheck Selbsttest" |
| 536 | echo " System : $(uname -s) $(uname -r)" |
| 537 | echo " Konfiguration : $([ -f "$CONF" ] && echo "$CONF" || echo "(keine, Defaults aktiv)")" |
| 538 | echo " Ziele : $TARGETS" |
| 539 | echo " Familien : IPv$(echo "$FAMILIES" | sed 's/ /, IPv/')" |
| 540 | echo " Datenablage : $OUTDIR" |
| 541 | echo |
| 542 | |
| 543 | _fail=0 |
| 544 | for _t in curl awk date find; do |
| 545 | if have "$_t"; then |
| 546 | echo " ok $_t ($(command -v "$_t"))" |
| 547 | else |
| 548 | echo " FEHLT $_t" |
| 549 | _fail=1 |
| 550 | fi |
| 551 | done |
| 552 | |
| 553 | if have mtr; then |
| 554 | echo " ok mtr ($(command -v mtr))" |
| 555 | if [ "$(id -u)" -eq 0 ]; then |
| 556 | echo " ok laeuft als root, mtr kann Raw Sockets oeffnen" |
| 557 | elif sudo -n true 2>/dev/null; then |
| 558 | echo " ok passwortloses sudo verfuegbar, mtr wird darueber gestartet" |
| 559 | else |
| 560 | echo " WARNUNG weder root noch passwortloses sudo - mtr wird scheitern." |
| 561 | echo " Entweder per root/sudo starten oder eine sudoers-Regel anlegen" |
| 562 | echo " (siehe README, Abschnitt Rechte)." |
| 563 | _fail=1 |
| 564 | fi |
| 565 | else |
| 566 | echo " FEHLT mtr - es werden nur curl-Messungen aufgezeichnet." |
| 567 | echo " macOS: brew install mtr | Debian/Ubuntu: apt install mtr-tiny" |
| 568 | fi |
| 569 | |
| 570 | echo |
| 571 | for _fam in $FAMILIES; do |
| 572 | for _t in $TARGETS; do |
| 573 | if curl "-$_fam" -sS -I --connect-timeout 5 --max-time 10 \ |
| 574 | -o /dev/null "https://$_t$CURL_PATH" 2>/dev/null; then |
| 575 | echo " ok IPv$_fam Testabruf $_t erfolgreich" |
| 576 | else |
| 577 | echo " WARNUNG IPv$_fam Testabruf $_t fehlgeschlagen (genau das wuerde das Monitoring aufzeichnen)" |
| 578 | fi |
| 579 | done |
| 580 | done |
| 581 | |
| 582 | echo |
| 583 | if [ "$_fail" -eq 0 ]; then |
| 584 | echo "Selbsttest ohne Blocker abgeschlossen." |
| 585 | else |
| 586 | echo "Selbsttest mit Problemen abgeschlossen - siehe Meldungen oben." |
| 587 | fi |
| 588 | return "$_fail" |
| 589 | } |
| 590 | |
| 591 | # --------------------------------------------------------------------------- |
| 592 | # Signalbehandlung und Aufraeumen |
| 593 | # --------------------------------------------------------------------------- |
| 594 | |
| 595 | # Ctrl+C bzw. SIGTERM beenden das Script nicht mitten in einer Messung, weil |
| 596 | # eine abgebrochene Messung als Ausfall in der CSV landen und die Auswertung |
| 597 | # verfaelschen wuerde. Stattdessen wird ein Stopp-Wunsch vermerkt, die |
| 598 | # angefangene Messung verworfen und danach sauber beendet. |
| 599 | # Ein zweites Signal erzwingt den sofortigen Abbruch. |
| 600 | on_signal() { |
| 601 | SIGCOUNT=$(( SIGCOUNT + 1 )) |
| 602 | STOP=1 |
| 603 | |
| 604 | # Laufende Messprozesse beenden, damit der Abbruch sofort greift |
| 605 | [ -n "$SLEEP_PID" ] && kill -TERM "$SLEEP_PID" 2>/dev/null |
| 606 | [ -n "$CHILD_PID" ] && kill -TERM "$CHILD_PID" 2>/dev/null |
| 607 | |
| 608 | if [ "$SIGCOUNT" -ge 2 ]; then |
| 609 | printf '\nnetcheck: Abbruch erzwungen.\n' >&9 |
| 610 | log "STOP zweites Signal - sofortiger Abbruch" |
| 611 | cleanup |
| 612 | exit 130 |
| 613 | fi |
| 614 | |
| 615 | printf '\nnetcheck: Stopp angefordert. Laufende Messung wird verworfen (nicht in die CSV geschrieben).\n' >&9 |
| 616 | printf 'netcheck: nochmal Ctrl+C erzwingt den sofortigen Abbruch.\n' >&9 |
| 617 | log "STOP Signal empfangen - laufende Messung wird verworfen" |
| 618 | } |
| 619 | |
| 620 | cleanup() { |
| 621 | [ -n "$CHILD_PID" ] && kill -TERM "$CHILD_PID" 2>/dev/null |
| 622 | [ -n "$SLEEP_PID" ] && kill -TERM "$SLEEP_PID" 2>/dev/null |
| 623 | # angefangene Rohlogs der abgebrochenen Messung entfernen |
| 624 | rm -f "$RAW_DIR"/*/.tmp-* 2>/dev/null |
| 625 | [ "$LOCK_HELD" -eq 1 ] && rm -rf "$LOCKDIR" |
| 626 | return 0 |
| 627 | } |
| 628 | |
| 629 | install_traps() { |
| 630 | trap 'on_signal' INT TERM |
| 631 | trap 'cleanup' EXIT |
| 632 | } |
| 633 | |
| 634 | # --------------------------------------------------------------------------- |
| 635 | # Sperre gegen ueberlappende Laeufe |
| 636 | # --------------------------------------------------------------------------- |
| 637 | |
| 638 | acquire_lock() { |
| 639 | if mkdir "$LOCKDIR" 2>/dev/null; then |
| 640 | echo $$ >"$LOCKDIR/pid" |
| 641 | LOCK_HELD=1 |
| 642 | install_traps |
| 643 | return 0 |
| 644 | fi |
| 645 | _pid="$(cat "$LOCKDIR/pid" 2>/dev/null)" |
| 646 | if [ -n "$_pid" ] && kill -0 "$_pid" 2>/dev/null; then |
| 647 | log "SKIP vorheriger Lauf (PID $_pid) laeuft noch" |
| 648 | return 1 |
| 649 | fi |
| 650 | log "WARN verwaiste Sperre von PID ${_pid:-unbekannt} entfernt" |
| 651 | rm -rf "$LOCKDIR" |
| 652 | mkdir "$LOCKDIR" 2>/dev/null || return 1 |
| 653 | echo $$ >"$LOCKDIR/pid" |
| 654 | LOCK_HELD=1 |
| 655 | install_traps |
| 656 | return 0 |
| 657 | } |
| 658 | |
| 659 | # --------------------------------------------------------------------------- |
| 660 | # Einstiegspunkt |
| 661 | # --------------------------------------------------------------------------- |
| 662 | |
| 663 | mkdir -p "$CSV_DIR" "$RAW_DIR" || die "kann $OUTDIR nicht anlegen" |
| 664 | |
| 665 | case "${1:-run}" in |
| 666 | once) |
| 667 | VERBOSE=1 |
| 668 | acquire_lock || exit 0 |
| 669 | echo "netcheck $(date '+%Y-%m-%dT%H:%M:%S%z')" |
| 670 | echo "CSV: $CSV_DIR/netcheck-$(date '+%Y-%m').csv" |
| 671 | do_run |
| 672 | echo "CSV: $CSV_DIR/netcheck-$(date '+%Y-%m').csv" |
| 673 | ;; |
| 674 | run) |
| 675 | acquire_lock || exit 0 |
| 676 | do_run |
| 677 | ;; |
| 678 | daemon) |
| 679 | VERBOSE=1 |
| 680 | acquire_lock || die "es laeuft bereits eine Instanz" |
| 681 | log "START daemon, Intervall ${INTERVAL}s, Ziele: $TARGETS" |
| 682 | echo "netcheck laeuft. Beenden mit Ctrl+C oder: $0 stop" |
| 683 | echo "CSV: $CSV_DIR/netcheck-$(date '+%Y-%m').csv" |
| 684 | echo "Roh: $RAW_DIR (nur bei Auffaelligkeiten)" |
| 685 | while [ "$STOP" -eq 0 ]; do |
| 686 | _start="$(date '+%s')" |
| 687 | do_run |
| 688 | [ "$STOP" -eq 1 ] && break |
| 689 | |
| 690 | _elapsed=$(( $(date '+%s') - _start )) |
| 691 | _sleep=$(( INTERVAL - _elapsed )) |
| 692 | [ "$_sleep" -lt 5 ] && _sleep=5 |
| 693 | |
| 694 | # Schlafen im Hintergrund, damit ein Signal sofort greift und |
| 695 | # nicht bis zum Ende des Intervalls warten muss. |
| 696 | sleep "$_sleep" & |
| 697 | SLEEP_PID=$! |
| 698 | wait "$SLEEP_PID" 2>/dev/null |
| 699 | SLEEP_PID="" |
| 700 | done |
| 701 | log "STOP daemon beendet" |
| 702 | echo "netcheck beendet." |
| 703 | ;; |
| 704 | stop) |
| 705 | _pid="$(cat "$LOCKDIR/pid" 2>/dev/null)" |
| 706 | if [ -z "$_pid" ]; then |
| 707 | echo "netcheck: keine laufende Instanz gefunden (keine Sperre in $LOCKDIR)." |
| 708 | exit 0 |
| 709 | fi |
| 710 | if ! kill -0 "$_pid" 2>/dev/null; then |
| 711 | echo "netcheck: verwaiste Sperre von PID $_pid entfernt." |
| 712 | rm -rf "$LOCKDIR" |
| 713 | exit 0 |
| 714 | fi |
| 715 | echo "netcheck: sende Stoppsignal an PID $_pid ..." |
| 716 | kill -TERM "$_pid" 2>/dev/null |
| 717 | _n=0 |
| 718 | while [ "$_n" -lt 60 ] && kill -0 "$_pid" 2>/dev/null; do |
| 719 | sleep 1 |
| 720 | _n=$(( _n + 1 )) |
| 721 | done |
| 722 | if kill -0 "$_pid" 2>/dev/null; then |
| 723 | echo "netcheck: reagiert nicht, erzwinge Abbruch." |
| 724 | kill -KILL "$_pid" 2>/dev/null |
| 725 | sleep 1 |
| 726 | rm -rf "$LOCKDIR" |
| 727 | else |
| 728 | echo "netcheck: sauber beendet." |
| 729 | fi |
| 730 | ;; |
| 731 | summary) |
| 732 | do_summary "${2:-}" |
| 733 | ;; |
| 734 | selftest) |
| 735 | do_selftest |
| 736 | ;; |
| 737 | -h|--help|help) |
| 738 | sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//' |
| 739 | ;; |
| 740 | *) |
| 741 | die "unbekannter Befehl '${1}'. Bekannt: once, run, daemon, stop, summary, selftest" |
| 742 | ;; |
| 743 | esac |