dtoro
3d88f52988
disk_usage_check.sh built its mount list with `df`, which blocks on a wedged filesystem (stale NFS export, a stuck ZFS pool) — and that stalled the whole check past the scheduler's 30s budget, leaving host:hubris:4 perpetually down. Build the mount list from /proc/mounts (a read that never stats anything), and bound every per-mount `df` with `timeout 8` so a single stuck mount is skipped instead of hanging the probe. Degrades to plain `df` on hosts without `timeout`//proc/mounts (macOS), whose local mounts don't hang.
43 lines
1.7 KiB
Bash
Executable File
43 lines
1.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# disk_usage_check.sh — disk usage and inode usage per mountpoint.
|
|
set -euo pipefail
|
|
|
|
# `timeout` caps each df so a single hung/stale mountpoint (a stale NFS
|
|
# export, a wedged ZFS pool) can't stall the whole check — that hung the
|
|
# scheduler's 30s budget on hubris. Available on Linux (coreutils); absent on
|
|
# Darwin, whose local mounts don't hang, so it degrades to an empty prefix.
|
|
TO=""
|
|
if command -v timeout >/dev/null 2>&1; then TO="timeout 8"; fi
|
|
|
|
# Build the mount list WITHOUT statting anything: reading /proc/mounts never
|
|
# blocks the way `df` does on a stuck filesystem, so the enumeration itself
|
|
# can't hang. Fall back to `df` on hosts without /proc/mounts (macOS).
|
|
if [ -r /proc/mounts ]; then
|
|
MOUNTS=$(awk '$1 ~ /^\// && $2 !~ /^\/(snap|dev|proc|sys|run|private)/ {print $2}' /proc/mounts || true)
|
|
else
|
|
MOUNTS=$($TO df -k 2>/dev/null | awk 'NR>1 && $1 ~ /^\// && $NF !~ /^\/(snap|dev|proc|sys|run|private)/ {print $NF}' || true)
|
|
fi
|
|
FIRST=1
|
|
|
|
echo -n '{"health":"healthy","metrics":{'
|
|
for m in $MOUNTS; do
|
|
# Each df is bounded: a stuck mount times out and is skipped (LINE empty)
|
|
# rather than hanging the probe.
|
|
LINE=$($TO df -k "$m" 2>/dev/null | awk 'NR==2 {print $3, $4, $5, $7}' | tr -d '%' || true)
|
|
if [ -z "$LINE" ]; then continue; fi
|
|
USED=$(echo "$LINE" | awk '{print $1}')
|
|
FREE=$(echo "$LINE" | awk '{print $2}')
|
|
PCT=$(echo "$LINE" | awk '{print $3}')
|
|
|
|
INODE_LINE=$($TO df -i "$m" 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%' || echo "0")
|
|
INODE_PCT=$(echo "${INODE_LINE:-0}" | sed 's/-/0/')
|
|
|
|
KEY=$(echo "$m" | sed 's|/|_|g' | sed 's|^_||')
|
|
[ -z "$KEY" ] && KEY="root"
|
|
|
|
if [ $FIRST -eq 0 ]; then echo -n ','; fi
|
|
FIRST=0
|
|
echo -n "\"disk_${KEY}_pct\":$PCT,\"inode_${KEY}_pct\":$INODE_PCT"
|
|
done
|
|
echo '}}'
|