28 lignes
976 B
Bash
28 lignes
976 B
Bash
#!/bin/bash
|
|
# Lightweight GPU watchdog: tails the host kernel ring buffer (if mounted)
|
|
# and warns loudly if the known amdgpu MES-hang pattern shows up while
|
|
# sd-cli is generating. Purely informational: it never kills the process,
|
|
# it just makes the problem visible immediately instead of after the fact.
|
|
#
|
|
# Requires /dev/kmsg (or a mounted /host-dmesg log) to be readable inside
|
|
# the container. If it isn't available, this script exits quietly.
|
|
|
|
set -u
|
|
|
|
LOGFILE="${1:-/dev/kmsg}"
|
|
|
|
if [ ! -r "$LOGFILE" ]; then
|
|
echo "[watch-gpu] $LOGFILE not readable from inside the container, skipping GPU watchdog." >&2
|
|
exit 0
|
|
fi
|
|
|
|
echo "[watch-gpu] Monitoring $LOGFILE for amdgpu MES/reset issues..."
|
|
|
|
tail -F -n0 "$LOGFILE" 2>/dev/null | while read -r line; do
|
|
case "$line" in
|
|
*"MES failed to respond"*|*"GPU reset"*|*"ring"*"timeout"*|*"device wedged"*|*"soft lockup"*)
|
|
echo "[watch-gpu] !!! GPU issue detected: $line" >&2
|
|
;;
|
|
esac
|
|
done
|