From 3d88f5298803e02beeeb29953d8bdaef37a16e02 Mon Sep 17 00:00:00 2001
From: dtoro <d.toro.v@pm.me>
Date: Wed, 29 Jul 2026 19:31:30 +0200
Subject: [PATCH] fix(checks): disk_usage_check no longer hangs on a stuck
 mount
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

disk_usage_check.sh built its mount list with `df`, which blocks on a wedged
filesystem (stale NFS export, a stuck ZFS pool) — and that stalled the whole
check past the scheduler's 30s budget, leaving host:hubris:4 perpetually down.

Build the mount list from /proc/mounts (a read that never stats anything), and
bound every per-mount `df` with `timeout 8` so a single stuck mount is skipped
instead of hanging the probe. Degrades to plain `df` on hosts without
`timeout`//proc/mounts (macOS), whose local mounts don't hang.
---
 checks/disk_usage_check.sh | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)
 mode change 100644 => 100755 checks/disk_usage_check.sh

diff --git a/checks/disk_usage_check.sh b/checks/disk_usage_check.sh
old mode 100644
new mode 100755
index fb4a477..1e3f5f2
--- a/checks/disk_usage_check.sh
+++ b/checks/disk_usage_check.sh
@@ -2,18 +2,34 @@
 # disk_usage_check.sh — disk usage and inode usage per mountpoint.
 set -euo pipefail
 
-MOUNTS=$(df -k 2>/dev/null | awk 'NR>1 && $1 ~ /^\// && $NF !~ /^\/(snap|dev|proc|sys|run|private)/ {print $NF}' || true)
+# `timeout` caps each df so a single hung/stale mountpoint (a stale NFS
+# export, a wedged ZFS pool) can't stall the whole check — that hung the
+# scheduler's 30s budget on hubris. Available on Linux (coreutils); absent on
+# Darwin, whose local mounts don't hang, so it degrades to an empty prefix.
+TO=""
+if command -v timeout >/dev/null 2>&1; then TO="timeout 8"; fi
+
+# Build the mount list WITHOUT statting anything: reading /proc/mounts never
+# blocks the way `df` does on a stuck filesystem, so the enumeration itself
+# can't hang. Fall back to `df` on hosts without /proc/mounts (macOS).
+if [ -r /proc/mounts ]; then
+  MOUNTS=$(awk '$1 ~ /^\// && $2 !~ /^\/(snap|dev|proc|sys|run|private)/ {print $2}' /proc/mounts || true)
+else
+  MOUNTS=$($TO df -k 2>/dev/null | awk 'NR>1 && $1 ~ /^\// && $NF !~ /^\/(snap|dev|proc|sys|run|private)/ {print $NF}' || true)
+fi
 FIRST=1
 
 echo -n '{"health":"healthy","metrics":{'
 for m in $MOUNTS; do
-  LINE=$(df -k "$m" 2>/dev/null | awk 'NR==2 {print $3, $4, $5, $7}' | tr -d '%' || true)
+  # Each df is bounded: a stuck mount times out and is skipped (LINE empty)
+  # rather than hanging the probe.
+  LINE=$($TO df -k "$m" 2>/dev/null | awk 'NR==2 {print $3, $4, $5, $7}' | tr -d '%' || true)
   if [ -z "$LINE" ]; then continue; fi
   USED=$(echo "$LINE" | awk '{print $1}')
   FREE=$(echo "$LINE" | awk '{print $2}')
   PCT=$(echo "$LINE" | awk '{print $3}')
 
-  INODE_LINE=$(df -i "$m" 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%' || echo "0")
+  INODE_LINE=$($TO df -i "$m" 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%' || echo "0")
   INODE_PCT=$(echo "${INODE_LINE:-0}" | sed 's/-/0/')
 
   KEY=$(echo "$m" | sed 's|/|_|g' | sed 's|^_||')