#!/bin/bash # # recover_nexaiq_edge_gpu.sh # # Recovery tool for a VSaaS Edge appliance whose GPU stack has wedged -- # the classic symptom being "nvidia-smi hangs forever over SSH" while the # application containers sit dead or restarting. # # Design rule #1: nvidia-smi may NEVER return. Every NVIDIA call in this # script is wrapped in `timeout`, so the recovery itself cannot hang. # (This is the bug that stops the bundled monitor_nvidia.sh from self-healing.) # # The script diagnoses first, then climbs an escalating repair ladder, # stopping as soon as the GPU answers: # # L1 quiesce GPU consumers (containers, persistence daemon, stray procs) # L2 reload the nvidia kernel modules # L3 GPU reset (nvidia-smi -r, then PCIe remove/rescan) # L4 repair the driver (DKMS rebuild, then full pinned reinstall) # L5 declare a reboot / cold power cycle necessary # # ...then restores the container runtime and brings the production stack up. # # Usage: sudo bash recover_nexaiq_edge_gpu.sh [options] # See --help for options. # # Requires: Ubuntu 22.04, root. Safe to re-run; it resumes after a reboot. # # Deliberately no `set -e`: recovery must survive failing steps and keep going. set -o pipefail #======================================================================= # Configuration #======================================================================= # Must match Install_VSaaS_Edge.sh / monitor_nvidia.sh TARGET_DRIVER_VERSION="575.51.03" DEB_PACKAGE="cuda-repo-ubuntu2204-12-9-local_12.9.0-575.51.03-1_amd64.deb" DEB_PACKAGE_URL="https://developer.download.nvidia.com/compute/cuda/12.9.0/local_installers/$DEB_PACKAGE" CUDA_PIN_URL="https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-ubuntu2204.pin" TEMP_DIR="/tmp" # Storage mounts the edge stack cannot run without MOUNT_PATHS=("/media/vdata" "/media/dbdata") # How long to wait on any NVIDIA call before declaring it hung NVSMI_TIMEOUT=20 # Where collected logs are shipped for support. Every value is overridable from # the environment or the command line, so a different site never has to edit # this file. Prefer REMOTE_PASS in the environment or --pass-file over the # built-in default below. REMOTE_HOST="${REMOTE_HOST:-172.232.26.226}" REMOTE_USER="${REMOTE_USER:-logsync}" REMOTE_BASE="${REMOTE_BASE:-/media/edge_logs/copart-videodata}" REMOTE_PASS="${REMOTE_PASS:-Logsync24!}" SSH_PORT="${SSH_PORT:-22}" REMOTE_KEY="${REMOTE_KEY:-}" # if set and readable, key auth is used instead # Persisted state so a post-reboot re-run does not loop on driver reinstall STATE_DIR="/var/lib/vsaas-gpu-recovery" STATE_FILE="$STATE_DIR/state" BOLD='\033[1m'; NORMAL='\033[0m' RED='\033[31m'; GREEN='\033[32m'; YELLOW='\033[33m'; CYAN='\033[36m' # Runtime options BUNDLE_DIR="" ALLOW_REBOOT=false FORCE_REINSTALL=false DIAGNOSE_ONLY=false SKIP_SERVICES=false SERVICES_ONLY=false DISABLE_GSP=false UPLOAD_LOGS=true PASS_FILE="" TARBALL="" # Diagnostic findings, filled in by diagnose() GPU_PRESENT=false GPU_BDFS=() GPU_OFF_BUS=false NVSMI_STATE="unknown" # ok | hung | missing | error NVSMI_DRIVER="" LOADED_MOD_VERSION="" DISK_MOD_VERSION="" DKMS_OK=true SECUREBOOT_ON=false DSTATE_PROCS="" GPU_DSTATE_PIDS="" # D-state pids wedged specifically in the NVIDIA driver STALE_PROBE_PIDS="" # leaked nvidia-smi probes from earlier runs of this script NVSMI_PROBE_LATCHED=false SHUTDOWN_WILL_HANG=false # a graceful `reboot` cannot complete in this state GSP_FAULT=false # Xid 119/120: GPU firmware (GSP) stopped responding MODULE_STATE_CORRUPT=false # a module refcount went negative: unload is impossible XID_CODES="" REPAIRED=false REBOOT_REQUIRED=false COLD_BOOT_REQUIRED=false #======================================================================= # Logging #======================================================================= LOG_DIR="./logs/recoverylogs" LOG_FILE="" TRACE_FILE="" # one line per guarded command: rc + duration DIAG_DIR="" log() { local ts; ts="$(date +'%Y-%m-%d %H:%M:%S')" while IFS= read -r line; do if [ -n "$LOG_FILE" ]; then echo -e "${ts} : ${line}" | tee -a "$LOG_FILE" else echo -e "${ts} : ${line}" fi done <<< "$1" } log_info() { log "$1"; } log_ok() { log "${BOLD}${GREEN}$1${NORMAL}"; } log_warn() { log "${BOLD}${YELLOW}$1${NORMAL}"; } log_err() { log "${BOLD}${RED}$1${NORMAL}"; } log_head() { log "#==================================================" log "# ${BOLD}${CYAN}$1${NORMAL}" log "#--------------------------------------------------" } # Run a command, sending its noisy output to the log file only. quiet() { "$@" >>"$LOG_FILE" 2>&1; } # One line per guarded command so a support engineer can see exactly what ran, # what it returned, and how long it blocked for. _trace() { [ -n "$TRACE_FILE" ] || return 0 printf '%s | rc=%-4s | %4ss | %s\n' \ "$(date +'%H:%M:%S')" "$1" "$2" "$3" >> "$TRACE_FILE" 2>/dev/null } #======================================================================= # Timeout-guarded helpers -- the core safety mechanism #======================================================================= # run_guarded # # Run a command that may wedge in uninterruptible (D) sleep and come back # regardless. Echoes the command's combined output on stdout. # Returns the command's exit status, or 124 if it had to be abandoned. # # Why plain `timeout` is NOT enough here: # 1. A process blocked in the kernel on a dead GPU sits in D state, where # SIGKILL is not delivered. `timeout -s KILL` sends the signal and then # wait()s for a child that will never die -- so timeout hangs too. # 2. Command substitution, out=$(...), reads the pipe until EOF. The # unkillable process holds the write end open forever, so $( ) blocks # even after timeout has given up. # Either way the caller hangs -- which is the exact fault this script exists # to repair, reproduced one layer up. # # So: hand the command a FILE (never an inherited pipe), run the whole thing # detached in the background, and poll for a completion marker. Once the # deadline passes we simply walk away from the child rather than reaping it. run_guarded() { local secs=$1; shift local base out rcfile runner waited status cmdline started cmdline="$*" started=$SECONDS base=$(mktemp -u "${TEMP_DIR:-/tmp}/guarded.XXXXXX") out="$base.out"; rcfile="$base.rc" : >"$out" # setsid puts it in its own session, so an abandoned child is orphaned to # init instead of lingering in our process group. ( if command -v setsid >/dev/null 2>&1; then setsid timeout -s KILL "$secs" "$@" >"$out" 2>&1 else timeout -s KILL "$secs" "$@" >"$out" 2>&1 fi # Write-then-rename so the poll below never sees a half-written file. echo $? >"$rcfile.tmp" && mv -f "$rcfile.tmp" "$rcfile" ) /dev/null 2>&1 & runner=$! # Deadline = the command's own timeout plus grace for timeout to give up. waited=0 while [ ! -f "$rcfile" ]; do if [ "$waited" -ge $((secs + 10)) ]; then kill -KILL "$runner" 2>/dev/null cat "$out" 2>/dev/null rm -f "$out" "$rcfile" "$rcfile.tmp" 2>/dev/null _trace "ABANDON" "$((SECONDS - started))" "$cmdline" return 124 # abandoned, deliberately not reaped fi sleep 1 waited=$((waited + 1)) done status=$(cat "$rcfile" 2>/dev/null) cat "$out" 2>/dev/null rm -f "$out" "$rcfile" "$rcfile.tmp" 2>/dev/null _trace "${status:-?}" "$((SECONDS - started))" "$cmdline" return "${status:-124}" } # nvsmi # Returns: 0 ok, 124/137 hung, 127 missing, other = real error. # Never blocks longer than NVSMI_TIMEOUT + 10s. nvsmi() { command -v nvidia-smi >/dev/null 2>&1 || return 127 run_guarded "$NVSMI_TIMEOUT" nvidia-smi "$@" } # Probe the GPU and set NVSMI_STATE. Cheap query, hard timeout. probe_nvidia_smi() { local out rc # Once the driver has proven wedged, every further nvidia-smi we launch # wedges too -- in D state, unkillable, holding a reference on the nvidia # module for as long as the box stays up. Those leaked probes are precisely # what makes the later rmmod impossible, so re-probing sabotages our own # repair. Latch the verdict and stop spawning until a repair step has # actually changed something (see invalidate_nvsmi_probe). if [ "$NVSMI_PROBE_LATCHED" = true ] && [ "$NVSMI_STATE" = "hung" ]; then log_info "Not re-probing: nvidia-smi is known hung and each probe leaks an unkillable process." return 0 fi out=$(nvsmi --query-gpu=driver_version --format=csv,noheader); rc=$? case $rc in 0) NVSMI_DRIVER=$(echo "$out" | head -n1 | tr -d '[:space:]') if [ -n "$NVSMI_DRIVER" ]; then NVSMI_STATE="ok" else NVSMI_STATE="error" fi ;; 124|137) NVSMI_STATE="hung" # This probe is now itself an unkillable D-state process. NVSMI_PROBE_LATCHED=true ;; 127) NVSMI_STATE="missing" ;; *) NVSMI_STATE="error" NVSMI_DRIVER="" log_info "nvidia-smi returned rc=$rc: $(echo "$out" | head -n3)" ;; esac return 0 } # Allow one fresh probe again -- call this only after a repair step that could # plausibly change the answer (module reload, GPU reset, driver reinstall). invalidate_nvsmi_probe() { NVSMI_PROBE_LATCHED=false; } # True when the GPU is genuinely answering queries. gpu_healthy() { probe_nvidia_smi [ "$NVSMI_STATE" = "ok" ] } #======================================================================= # Preflight #======================================================================= usage() { cat <<'USAGE' Usage: sudo bash recover_nexaiq_edge_gpu.sh [options] Recovers a VSaaS Edge box whose NVIDIA stack has hung (nvidia-smi never returns) and brings the production container stack back up. Options: --bundle-dir DIR Edge bundle containing docker-compose.yml. Default: current directory, else autodetected. --diagnose-only Collect diagnostics and report. Changes nothing. --allow-reboot Permit the script to reboot when a repair needs it. Without this it stops and tells you to reboot. --force-reinstall Skip the ladder; go straight to a full pinned reinstall of driver 575.51.03. --skip-services Repair the GPU but do not touch docker compose. --services-only Skip GPU repair entirely; just bring the container stack back up. Use after a power cycle when the GPU is healthy and you only need the box operational. --disable-gsp Persistently disable GSP firmware (NVreg_EnableGpuFirmware=0). Use when the box keeps hitting Xid 119 GSP RPC timeouts. Takes effect on the next boot. Log upload (runs automatically once the logs have been collected): --no-upload Do not ship anything. All logs stay on this box. --remote-host HOST Default: 172.232.26.226 --remote-user USER Default: logsync --remote-base PATH Default: /media/edge_logs/copart-videodata --ssh-port N Default: 22 --ssh-key FILE Use SSH key auth instead of a password (preferred). --pass-file FILE Read the password from the first line of FILE instead of embedding it. The REMOTE_PASS environment variable works too; both override the built-in default. --timeout N Seconds before an NVIDIA call is called hung (default 20). -h, --help This message. Exit codes: 0 recovered / healthy, 1 fatal, 2 reboot required, 3 hardware fault. USAGE } parse_args() { while [ $# -gt 0 ]; do case "$1" in --bundle-dir) BUNDLE_DIR="$2"; shift 2 ;; --diagnose-only) DIAGNOSE_ONLY=true; shift ;; --allow-reboot) ALLOW_REBOOT=true; shift ;; --force-reinstall) FORCE_REINSTALL=true; shift ;; --skip-services) SKIP_SERVICES=true; shift ;; --services-only) SERVICES_ONLY=true; shift ;; --no-upload) UPLOAD_LOGS=false; shift ;; --pass-file) PASS_FILE="$2"; shift 2 ;; --remote-host) REMOTE_HOST="$2"; shift 2 ;; --remote-user) REMOTE_USER="$2"; shift 2 ;; --remote-base) REMOTE_BASE="$2"; shift 2 ;; --ssh-port) SSH_PORT="$2"; shift 2 ;; --ssh-key) REMOTE_KEY="$2"; shift 2 ;; --disable-gsp) DISABLE_GSP=true; shift ;; --timeout) NVSMI_TIMEOUT="$2"; shift 2 ;; -h|--help) usage; exit 0 ;; *) echo "Unknown option: $1"; usage; exit 1 ;; esac done } # Locate the deployment bundle (the dir holding docker-compose.yml). resolve_bundle_dir() { if [ -n "$BUNDLE_DIR" ]; then if [ ! -f "$BUNDLE_DIR/docker-compose.yml" ]; then log_err "No docker-compose.yml in --bundle-dir '$BUNDLE_DIR'." exit 1 fi BUNDLE_DIR=$(cd "$BUNDLE_DIR" && pwd) return fi if [ -f "./docker-compose.yml" ]; then BUNDLE_DIR=$(pwd) return fi # Autodetect: a bundle is a dir with both docker-compose.yml and the installer. local found found=$(find /home /opt /srv -maxdepth 4 -name docker-compose.yml \ -path '*edge*' -printf '%h\n' 2>/dev/null | head -n1) if [ -n "$found" ] && [ -f "$found/Install_VSaaS_Edge.sh" ]; then BUNDLE_DIR="$found" log_warn "Autodetected bundle directory: $BUNDLE_DIR" else log_warn "Could not locate a deployment bundle; service restore will be skipped." log_warn "Re-run with --bundle-dir /path/to/bundle to restore services." SKIP_SERVICES=true BUNDLE_DIR=$(pwd) fi } preflight() { if [ "$(id -u)" -ne 0 ]; then echo "Please run this script with sudo." exit 1 fi resolve_bundle_dir cd "$BUNDLE_DIR" || exit 1 mkdir -p "$LOG_DIR" "$STATE_DIR" # Make the log path ABSOLUTE. LOG_DIR is relative to the bundle directory, # which this function has just cd'd into -- and that is frequently not the # directory the operator ran the script from. Printing "./logs/..." then # sends them looking in the wrong place and it reads as "no logs were # generated". Resolve it once, here, and report absolute paths everywhere. LOG_DIR=$(cd "$LOG_DIR" 2>/dev/null && pwd) || LOG_DIR="$BUNDLE_DIR/logs/recoverylogs" # One timestamp for all three, so the log, the trace and the diagnostics # directory always share a name even if the clock ticks mid-preflight. local stamp; stamp=$(date +'%Y%m%d_%H%M%S') LOG_FILE="$LOG_DIR/gpu_recovery_$stamp.log" TRACE_FILE="$LOG_DIR/gpu_recovery_${stamp}_trace.log" DIAG_DIR="$LOG_DIR/diagnostics_$stamp" mkdir -p "$DIAG_DIR" # Create both files immediately so they exist even if the run dies early. : > "$LOG_FILE" : > "$TRACE_FILE" if [ ! -w "$LOG_FILE" ]; then echo "WARNING: cannot write $LOG_FILE -- output will be terminal-only." >&2 LOG_FILE="" fi log "===================================================================================" log "VSaaS Edge GPU recovery -- $(date)" log "Bundle: $BUNDLE_DIR" log "Logs dir: $LOG_DIR" log "Run log: $LOG_FILE" log "Cmd trace: $TRACE_FILE" log "Diagnostics: $DIAG_DIR" log "===================================================================================" local os_name os_version if [ -f /etc/os-release ]; then os_name=$(. /etc/os-release; echo "$NAME") os_version=$(. /etc/os-release; echo "$VERSION_ID") if [ "$os_name" = "Ubuntu" ] && [ "$os_version" = "22.04" ]; then log_ok "OS check passed: $os_name $os_version" else # Warn but continue -- refusing to run helps nobody mid-outage. log_warn "Unsupported OS: $os_name $os_version (expected Ubuntu 22.04). Continuing anyway." fi fi for cmd in timeout lspci dmesg modprobe; do command -v "$cmd" >/dev/null 2>&1 || log_warn "Missing required tool: $cmd" done } #======================================================================= # PHASE 1 -- Diagnostics #======================================================================= diagnose() { log_head "PHASE 1: Diagnostics" # --- GPU visible on the PCI bus? ----------------------------------- local lspci_out lspci_out=$(lspci -nn 2>/dev/null | grep -i nvidia) echo "$lspci_out" > "$DIAG_DIR/lspci.txt" if [ -n "$lspci_out" ]; then GPU_PRESENT=true log_ok "NVIDIA device(s) present on PCI bus:" log_info "$lspci_out" mapfile -t GPU_BDFS < <(lspci -D 2>/dev/null | grep -i nvidia | awk '{print $1}') # "rev ff" means the device stopped responding -- fallen off the bus. if echo "$lspci_out" | grep -q "rev ff"; then GPU_OFF_BUS=true log_err "GPU reports PCI revision ff -- it has FALLEN OFF THE BUS." fi else log_err "No NVIDIA device found by lspci. GPU is not visible to the system." fi # --- Does nvidia-smi answer? (timeout-guarded) --------------------- log_info "Probing nvidia-smi (max ${NVSMI_TIMEOUT}s)..." probe_nvidia_smi case "$NVSMI_STATE" in ok) log_ok "nvidia-smi responded. Driver version: $NVSMI_DRIVER" ;; hung) log_err "nvidia-smi HUNG (no response in ${NVSMI_TIMEOUT}s) -- this is the reported fault." ;; missing) log_err "nvidia-smi is not installed." ;; error) log_err "nvidia-smi failed to talk to the driver." ;; esac # --- Kernel ring buffer: Xid errors -------------------------------- dmesg -T 2>/dev/null | grep -iE 'nvrm|nvidia|xid' | tail -n 200 > "$DIAG_DIR/dmesg_nvidia.txt" # Xid lines read: "NVRM: Xid (PCI:0000:01:00): 79, pid=..., " # Anchor on the ": ," that follows the PCI address, or a bare "Xid 79:". XID_CODES=$( { sed -nE 's/.*Xid[^,]*[:,][[:space:]]*([0-9]+),.*/\1/p' "$DIAG_DIR/dmesg_nvidia.txt" sed -nE 's/.*Xid[[:space:]]+([0-9]+)[:,].*/\1/p' "$DIAG_DIR/dmesg_nvidia.txt" } 2>/dev/null | sort -un | tr '\n' ' ') if [ -n "$XID_CODES" ]; then log_warn "NVRM Xid errors found in dmesg: $XID_CODES" interpret_xids "$XID_CODES" else log_info "No Xid errors in the current kernel ring buffer." fi # --- Driver version: loaded module vs on-disk module vs package ---- # A mismatch here is the #1 cause after an unattended apt upgrade: # the .ko files on disk were replaced but the old module is still resident. if [ -r /proc/driver/nvidia/version ]; then # Reading this proc file also blocks in D state when the driver is wedged. LOADED_MOD_VERSION=$(run_guarded 5 cat /proc/driver/nvidia/version 2>/dev/null \ | grep -oE 'Kernel Module +[0-9.]+' | awk '{print $3}') fi if command -v modinfo >/dev/null 2>&1; then DISK_MOD_VERSION=$(modinfo nvidia 2>/dev/null | awk '/^version:/{print $2}') fi log_info "Kernel module loaded in memory : ${LOADED_MOD_VERSION:-}" log_info "Kernel module on disk : ${DISK_MOD_VERSION:-}" log_info "Target driver version : $TARGET_DRIVER_VERSION" if [ -n "$LOADED_MOD_VERSION" ] && [ -n "$DISK_MOD_VERSION" ] \ && [ "$LOADED_MOD_VERSION" != "$DISK_MOD_VERSION" ]; then log_err "DRIVER VERSION MISMATCH: running module $LOADED_MOD_VERSION vs on-disk $DISK_MOD_VERSION." log_err "The driver was upgraded under a running kernel. A module reload or reboot is required." REBOOT_REQUIRED=true fi # --- DKMS built against the running kernel? ------------------------ if command -v dkms >/dev/null 2>&1; then dkms status > "$DIAG_DIR/dkms_status.txt" 2>&1 log_info "dkms status: $(tr '\n' '; ' < "$DIAG_DIR/dkms_status.txt")" if ! grep -q "$(uname -r)" "$DIAG_DIR/dkms_status.txt" 2>/dev/null; then if grep -qi nvidia "$DIAG_DIR/dkms_status.txt" 2>/dev/null; then DKMS_OK=false log_err "NVIDIA DKMS module is NOT built for the running kernel $(uname -r)." fi fi fi log_info "Running kernel: $(uname -r)" # --- Secure Boot blocks unsigned modules --------------------------- if command -v mokutil >/dev/null 2>&1; then if mokutil --sb-state 2>/dev/null | grep -qi enabled; then SECUREBOOT_ON=true log_warn "Secure Boot is ENABLED. Unsigned NVIDIA modules will refuse to load." fi fi # --- Device nodes -------------------------------------------------- ls -l /dev/nvidia* > "$DIAG_DIR/dev_nodes.txt" 2>&1 if ls /dev/nvidia0 >/dev/null 2>&1; then log_info "Device nodes present: $(ls /dev/nvidia* 2>/dev/null | tr '\n' ' ')" else log_warn "No /dev/nvidia0 device node -- the driver is not bound to the GPU." fi # --- Module reference counts --------------------------------------- # A negative refcount is the fingerprint of an rmmod that was interrupted # partway through (ours, at 20:02, wedged in D state doing exactly this). # The module can never be unloaded again, so no ladder rung can succeed. local m rcv for m in nvidia nvidia_modeset nvidia_drm nvidia_uvm; do [ -r "/sys/module/$m/refcnt" ] || continue rcv=$(cat "/sys/module/$m/refcnt" 2>/dev/null) case "$rcv" in ''|*[!0-9-]*) continue ;; esac if [ "$rcv" -lt 0 ]; then MODULE_STATE_CORRUPT=true log_err "Module $m has a CORRUPTED reference count ($rcv)." log_err "An earlier module unload was interrupted. $m can never be removed" log_err "again on this boot, so no software repair can succeed." fi done # --- Loaded modules and their reference counts --------------------- lsmod | grep -i nvidia > "$DIAG_DIR/lsmod_nvidia.txt" 2>&1 if [ -s "$DIAG_DIR/lsmod_nvidia.txt" ]; then log_info "Loaded NVIDIA modules:" log_info "$(cat "$DIAG_DIR/lsmod_nvidia.txt")" else log_warn "No NVIDIA kernel modules are loaded." fi # --- Uninterruptible (D-state) processes --------------------------- # A process stuck in D state on the GPU cannot be killed. If one exists, # the module cannot be unloaded and only a reboot will clear it. DSTATE_PROCS=$(ps -eo pid,stat,comm,args --no-headers 2>/dev/null \ | awk '$2 ~ /^D/ {print}') echo "$DSTATE_PROCS" > "$DIAG_DIR/dstate_procs.txt" if [ -n "$DSTATE_PROCS" ]; then log_err "Processes stuck in uninterruptible (D) state -- unkillable, reboot likely required:" log_err "$DSTATE_PROCS" # Only D-state tasks wedged in the NVIDIA driver pin the module. One # stuck on, say, an unresponsive NFS mount is irrelevant here, so do # not let it veto the repair ladder. GPU_DSTATE_PIDS=$(echo "$DSTATE_PROCS" \ | awk '$3 ~ /nvidia|nv_/ || $0 ~ /nvidia/ {print $1}' \ | tr '\n' ' ') [ -n "$GPU_DSTATE_PIDS" ] && log_err "Wedged in the NVIDIA driver: $GPU_DSTATE_PIDS" # Our own abandoned probes look exactly like this. Name them, because # they are self-inflicted and they are what blocks rmmod. STALE_PROBE_PIDS=$(echo "$DSTATE_PROCS" \ | awk '/nvidia-smi/ && /query-gpu=driver_version/ {print $1}' \ | tr '\n' ' ') if [ -n "$GPU_DSTATE_PIDS" ]; then # systemd's shutdown waits for every process to exit. A D-state task # never does, so `reboot` stalls at "Reached target Shutdown" and the # machine never actually restarts -- which is why a power cycle ends # up being the only way back. SHUTDOWN_WILL_HANG=true log_err "A graceful 'sudo reboot' will HANG with these processes present:" log_err "systemd cannot terminate them, so shutdown never completes." fi if [ -n "$STALE_PROBE_PIDS" ]; then log_err "PIDs $STALE_PROBE_PIDS are leaked nvidia-smi probes from earlier runs of this" log_err "script. They cannot be killed and they hold the nvidia module open, so no" log_err "amount of re-running will unload it -- only a reboot clears them." fi else log_info "No processes stuck in D state." fi # --- Who is holding the GPU device nodes? -------------------------- if command -v fuser >/dev/null 2>&1 && ls /dev/nvidia* >/dev/null 2>&1; then # fuser opens the device nodes, so it wedges on a dead GPU exactly like nvidia-smi. run_guarded 15 fuser -v /dev/nvidia* > "$DIAG_DIR/gpu_holders.txt" 2>&1 log_info "Processes holding /dev/nvidia*: $(grep -c . "$DIAG_DIR/gpu_holders.txt" 2>/dev/null) line(s), see diagnostics." fi # --- Kernel taint / hardware errors --------------------------------- log_info "Kernel taint flags: $(cat /proc/sys/kernel/tainted 2>/dev/null)" dmesg -T 2>/dev/null | grep -iE 'hardware error|mce|pcieport.*error|thermal' | tail -n 40 \ > "$DIAG_DIR/dmesg_hw_errors.txt" if [ -s "$DIAG_DIR/dmesg_hw_errors.txt" ]; then log_warn "Hardware/PCIe/thermal errors present in dmesg -- see $DIAG_DIR/dmesg_hw_errors.txt" fi # --- Storage mounts the stack requires ------------------------------ for mp in "${MOUNT_PATHS[@]}"; do if mountpoint -q "$mp"; then log_ok "$mp is mounted." else log_err "$mp is NOT mounted -- the edge stack cannot run without it." fi done # --- Disk space (a full disk mimics an application hang) ------------ df -h > "$DIAG_DIR/df.txt" 2>&1 local full full=$(df -h --output=pcent,target 2>/dev/null | awk 'NR>1 && $1+0 >= 90 {print $2" "$1}') if [ -n "$full" ]; then log_warn "Filesystems at or above 90% capacity: $full" fi # --- Docker daemon state -------------------------------------------- if systemctl is-active --quiet docker; then log_ok "Docker daemon is running." else log_warn "Docker daemon is NOT running." fi collect_diagnostics } #======================================================================= # Detailed diagnostic collection #======================================================================= # dump # Run one collector, write its combined output into the diagnostics tree, and # note what happened. Never aborts the run, however badly the collector behaves. dump() { local name=$1 secs=$2; shift 2 local target="$DIAG_DIR/$name" rc mkdir -p "$(dirname "$target")" 2>/dev/null if ! command -v "$1" >/dev/null 2>&1; then echo "(collector '$1' is not installed on this system)" > "$target" return 0 fi run_guarded "$secs" "$@" > "$target" 2>&1 rc=$? case $rc in 0) log_info " $name" ;; 124|137) log_warn " $name -- TIMED OUT after ${secs}s (collector wedged)" ;; *) log_info " $name (rc=$rc)" ;; esac return 0 } # Write a here-doc style file without spawning anything. note() { printf '%s\n' "$2" > "$DIAG_DIR/$1" 2>/dev/null; } collect_diagnostics() { log_head "Collecting detailed diagnostics" log_info " Target: $DIAG_DIR" #--- GPU --------------------------------------------------------------- log_info " [gpu]" dump gpu/lspci_nvidia.txt 15 lspci -nnk -d 10de: dump gpu/lspci_tree.txt 15 lspci -tv dump gpu/lspci_verbose.txt 30 sh -c 'for b in $(lspci -D | grep -i nvidia | awk "{print \$1}"); do echo "=== $b ==="; lspci -vvv -s "$b" 2>&1; done' dump gpu/dev_nodes.txt 10 ls -l /dev/nvidia /dev/nvidia0 /dev/nvidiactl /dev/nvidia-uvm /dev/nvidia-caps dump gpu/proc_driver_version.txt 10 cat /proc/driver/nvidia/version dump gpu/proc_gpus.txt 15 sh -c 'for g in /proc/driver/nvidia/gpus/*/information; do echo "=== $g ==="; cat "$g" 2>&1; done' dump gpu/proc_params.txt 10 cat /proc/driver/nvidia/params dump gpu/modprobe_conf.txt 10 sh -c 'grep -r . /etc/modprobe.d/ 2>&1 | grep -i nvidia' # Anything that talks to the card is only safe when the card is answering. # On a wedged GPU these collectors block in D state forever and each one # leaves another unkillable process pinning the driver (see nvsmi/run_guarded). if [ "$NVSMI_STATE" = "ok" ]; then dump gpu/nvidia-smi_q.txt 60 nvidia-smi -q dump gpu/nvidia-smi_topo.txt 30 nvidia-smi topo -m dump gpu/nvidia-smi_procs.txt 30 nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv dump gpu/nvidia-smi_ecc.txt 30 nvidia-smi -q -d ECC,TEMPERATURE,POWER,CLOCK,PERFORMANCE if command -v nvidia-bug-report.sh >/dev/null 2>&1; then log_info " running nvidia-bug-report.sh (this is slow)..." ( cd "$DIAG_DIR/gpu" 2>/dev/null && run_guarded 180 nvidia-bug-report.sh >/dev/null 2>&1 ) [ -f "$DIAG_DIR/gpu/nvidia-bug-report.log.gz" ] \ && log_ok " nvidia-bug-report.log.gz captured" \ || log_warn " nvidia-bug-report.sh produced nothing (likely wedged)" fi else note gpu/SKIPPED_nvidia-smi.txt \ "nvidia-smi collectors were deliberately SKIPPED. NVSMI_STATE was '$NVSMI_STATE', i.e. the driver is not answering. Every nvidia-smi invocation against a wedged driver blocks in uninterruptible (D) state, cannot be killed, and holds a reference on the nvidia module for the rest of this boot -- which is precisely what prevents the module from being unloaded and the box from shutting down cleanly. Collecting them would have made the fault worse, so they were not run. See dmesg_nvidia.txt and xid_history.txt for what the driver reported instead." log_warn " nvidia-smi collectors skipped (GPU wedged -- would leak D-state procs)" fi #--- Kernel ------------------------------------------------------------ log_info " [kernel]" dump kernel/dmesg_full.txt 30 dmesg -T dump kernel/dmesg_nvidia.txt 20 sh -c 'dmesg -T | grep -iE "nvrm|nvidia|xid|gsp"' dump kernel/xid_history.txt 20 sh -c 'dmesg -T | grep -iE "xid" ; echo "--- journal, all boots ---"; journalctl -k --no-pager 2>/dev/null | grep -iE "xid"' dump kernel/dmesg_errors.txt 20 sh -c 'dmesg -T -l err,crit,alert,emerg' dump kernel/dmesg_hw.txt 20 sh -c 'dmesg -T | grep -iE "hardware error|mce|pcieport|thermal|throttl|power limit"' dump kernel/uname.txt 10 uname -a dump kernel/cmdline.txt 10 cat /proc/cmdline dump kernel/tainted.txt 10 cat /proc/sys/kernel/tainted dump kernel/modules.txt 10 lsmod dump kernel/module_refcounts.txt 15 sh -c 'for m in nvidia nvidia_modeset nvidia_drm nvidia_uvm; do [ -d /sys/module/$m ] && echo "$m refcnt=$(cat /sys/module/$m/refcnt 2>&1) holders=$(ls /sys/module/$m/holders 2>/dev/null | tr "\n" " ")"; done' dump kernel/dkms_status.txt 30 dkms status dump kernel/headers.txt 15 sh -c 'dpkg -l | grep -E "linux-headers|linux-image"' dump kernel/secureboot.txt 15 mokutil --sb-state dump kernel/interrupts.txt 10 cat /proc/interrupts dump kernel/iomem.txt 10 cat /proc/iomem #--- Processes / system ------------------------------------------------ log_info " [system]" dump system/ps_full.txt 20 ps -eo pid,ppid,stat,pcpu,pmem,etime,comm,args dump system/dstate_procs.txt 20 sh -c 'ps -eo pid,stat,comm,args --no-headers | awk "\$2 ~ /^D/"' dump system/dstate_stacks.txt 30 sh -c 'for p in $(ps -eo pid,stat --no-headers | awk "\$2 ~ /^D/ {print \$1}"); do echo "=== PID $p ($(cat /proc/$p/comm 2>/dev/null)) ==="; cat /proc/$p/stack 2>&1; echo; done' dump system/gpu_holders.txt 20 sh -c 'fuser -v /dev/nvidia* 2>&1' dump system/uptime.txt 10 uptime dump system/reboot_history.txt 15 last -x dump system/meminfo.txt 10 cat /proc/meminfo dump system/df.txt 15 df -h dump system/mounts.txt 10 findmnt -A dump system/sensors.txt 20 sensors dump system/failed_units.txt 30 systemctl list-units --state=failed --no-pager dump system/units_enabled.txt 30 systemctl list-unit-files --state=enabled --no-pager dump system/crontab_root.txt 10 crontab -l #--- Journal (previous boots matter: the box has been power-cycled) ----- log_info " [journal]" dump journal/boots.txt 20 journalctl --list-boots --no-pager dump journal/this_boot_kernel.txt 40 journalctl -k -b 0 --no-pager dump journal/prev_boot_kernel.txt 40 journalctl -k -b -1 --no-pager dump journal/prev_boot_tail.txt 40 sh -c 'journalctl -b -1 --no-pager | tail -n 3000' dump journal/this_boot_errors.txt 40 journalctl -b 0 -p err --no-pager dump journal/docker.txt 40 sh -c 'journalctl -u docker --no-pager | tail -n 2000' dump journal/shutdown.txt 40 sh -c 'journalctl -b -1 --no-pager | grep -iE "shutdown|reboot|unmount|stopping|watchdog|failed to" | tail -n 500' #--- Docker / stack ---------------------------------------------------- log_info " [docker]" dump docker/info.txt 30 docker info dump docker/version.txt 20 docker version dump docker/ps_all.txt 30 docker ps -a dump docker/images.txt 30 docker images dump docker/daemon_json.txt 10 cat /etc/docker/daemon.json dump docker/runtimes.txt 20 sh -c 'docker info 2>/dev/null | grep -iA5 runtime' dump docker/nvidia_ctk.txt 20 nvidia-ctk --version dump docker/compose_ps.txt 60 sh -c "cd '$BUNDLE_DIR' 2>/dev/null && docker compose ps -a" dump docker/compose_config.txt 60 sh -c "cd '$BUNDLE_DIR' 2>/dev/null && docker compose config" dump docker/container_logs.txt 120 sh -c 'for c in $(docker ps -a --format "{{.Names}}" 2>/dev/null); do echo "=================== $c ==================="; docker logs --tail 300 "$c" 2>&1; echo; done' #--- Packages ---------------------------------------------------------- log_info " [packages]" dump packages/nvidia_cuda.txt 20 sh -c 'dpkg -l | grep -iE "nvidia|cuda|libnv"' dump packages/holds.txt 15 apt-mark showhold dump packages/apt_history.txt 20 sh -c 'cat /var/log/apt/history.log 2>/dev/null; zcat /var/log/apt/history.log.*.gz 2>/dev/null' dump packages/unattended.txt 20 sh -c 'cat /var/log/unattended-upgrades/unattended-upgrades.log 2>/dev/null | tail -n 500' dump packages/sources.txt 15 sh -c 'grep -rs . /etc/apt/sources.list /etc/apt/sources.list.d/ 2>/dev/null' #--- Bundle state ------------------------------------------------------ log_info " [bundle]" dump bundle/listing.txt 20 ls -la "$BUNDLE_DIR" dump bundle/engines.txt 20 ls -la "$BUNDLE_DIR/engines" dump bundle/pro_cnf.txt 10 cat "$BUNDLE_DIR/vtpl_cnf/pro.cnf" dump bundle/recovery_state.txt 10 cat "$STATE_FILE" #--- Findings summary -------------------------------------------------- note 00-SUMMARY.txt \ "VSaaS Edge GPU recovery -- diagnostic summary ============================================= Collected : $(date) Host : $(hostname 2>/dev/null) Bundle : $BUNDLE_DIR Kernel : $(uname -r) Script : $0 FINDINGS -------- GPU present on PCI bus : $GPU_PRESENT GPU fallen off bus : $GPU_OFF_BUS nvidia-smi state : $NVSMI_STATE Driver reported by smi : ${NVSMI_DRIVER:-} Kernel module (loaded) : ${LOADED_MOD_VERSION:-} Kernel module (on disk) : ${DISK_MOD_VERSION:-} Target driver version : $TARGET_DRIVER_VERSION DKMS built for kernel : $DKMS_OK Secure Boot enabled : $SECUREBOOT_ON Xid codes seen : ${XID_CODES:-} GSP firmware fault : $GSP_FAULT Module state corrupted : $MODULE_STATE_CORRUPT D-state pids (in driver): ${GPU_DSTATE_PIDS:-} Leaked probe pids : ${STALE_PROBE_PIDS:-} Graceful reboot hangs : $SHUTDOWN_WILL_HANG Reboot required : $REBOOT_REQUIRED Cold power cycle needed : $COLD_BOOT_REQUIRED WHERE TO LOOK ------------- kernel/xid_history.txt Xid codes across this and previous boots kernel/module_refcounts.txt negative refcnt => module can never be unloaded system/dstate_stacks.txt kernel stack of every unkillable task journal/prev_boot_kernel.txt what the box logged before the last power cycle journal/shutdown.txt why a graceful shutdown did not complete gpu/SKIPPED_nvidia-smi.txt present when GPU collectors were skipped, and why ../gpu_recovery_*_trace.log every guarded command with rc and duration" log_ok " Detailed diagnostics collected under $DIAG_DIR" } # Translate the Xid codes we found into operator-readable causes. interpret_xids() { local codes="$1" c for c in $codes; do case "$c" in 13) log_warn " Xid 13: graphics engine exception (usually an application bug)." ;; 31) log_warn " Xid 31: GPU memory page fault (application bug)." ;; 43) log_warn " Xid 43: GPU stopped processing (application-level error)." ;; 45) log_warn " Xid 45: preemptive cleanup / process killed while on GPU." ;; 48) log_err " Xid 48: DOUBLE-BIT ECC ERROR -- hardware fault. GPU likely needs replacement." ;; 62) log_err " Xid 62: internal micro-controller halt. GPU reset required." ;; 63|64) log_err " Xid 63/64: ECC page retirement/remapping event. Monitor GPU health." ;; 74) log_err " Xid 74: NVLink/PCIe error. Check seating and riser cabling." ;; 79) log_err " Xid 79: GPU HAS FALLEN OFF THE BUS -- needs a full cold power cycle, not a warm reboot." COLD_BOOT_REQUIRED=true ;; 94|95) log_err " Xid 94/95: contained/uncontained ECC error." ;; 119|120) log_err " Xid $c: GSP RPC TIMEOUT -- the GPU's GSP firmware has stopped answering" log_err " the driver. Every nvidia-smi then blocks in the kernel forever." log_err " A warm reboot does NOT reset the GSP: the card keeps its state" log_err " across it, so the fault survives and the box comes back broken." log_err " Only removing power resets the GSP." GSP_FAULT=true COLD_BOOT_REQUIRED=true ;; *) log_warn " Xid $c: consult NVIDIA Xid documentation." ;; esac done } #======================================================================= # PHASE 2 -- Repair ladder #======================================================================= # L1: stop everything that could be holding the GPU. quiesce_gpu_consumers() { log_info "[L1] Stopping GPU consumers..." if [ -f "$BUNDLE_DIR/docker-compose.yml" ] && systemctl is-active --quiet docker; then log_info "Stopping the edge container stack..." ( cd "$BUNDLE_DIR" && run_guarded 180 docker compose down >>"$LOG_FILE" 2>&1 ) fi # A display manager holds nvidia_drm open and will block the module unload. for svc in gdm3 lightdm sddm; do if systemctl is-active --quiet "$svc" 2>/dev/null; then log_warn "Stopping display manager $svc (it pins nvidia_drm)..." quiet systemctl stop "$svc" fi done for svc in nvidia-persistenced nvidia-fabricmanager; do if systemctl list-unit-files 2>/dev/null | grep -q "^${svc}"; then log_info "Stopping $svc..." quiet systemctl stop "$svc" fi done # Kill anything still holding a GPU device node (D-state procs will resist). if command -v fuser >/dev/null 2>&1 && ls /dev/nvidia* >/dev/null 2>&1; then local holders holders=$(run_guarded 15 fuser /dev/nvidia* 2>/dev/null | tr -s ' ' '\n' | grep -E '^[0-9]+$' | sort -u) if [ -n "$holders" ]; then # Signalling a D-state task does nothing at all; listing it as # "terminated" would just be a lie in the log. Split them out. local killable="" p for p in $holders; do case " $GPU_DSTATE_PIDS " in *" $p "*) log_warn " PID $p is in uninterruptible (D) state -- cannot be signalled, skipping." ;; *) killable="$killable $p" ;; esac done if [ -n "$killable" ]; then log_warn "Terminating processes holding the GPU:$killable" # shellcheck disable=SC2086 kill -TERM $killable 2>/dev/null sleep 5 # shellcheck disable=SC2086 kill -KILL $killable 2>/dev/null sleep 2 fi fi fi } # L2: unload and reload the kernel modules. Order matters -- dependants first. reload_nvidia_modules() { log_info "[L2] Reloading NVIDIA kernel modules..." if [ -n "$DSTATE_PROCS" ]; then log_warn "D-state processes are present; module unload will probably fail." fi # Docker/containerd keep device references open; stop them for the unload. local docker_was_running=false if systemctl is-active --quiet docker; then docker_was_running=true log_info "Stopping docker and containerd to release GPU device handles..." quiet systemctl stop docker.socket quiet systemctl stop docker quiet systemctl stop containerd fi # rmmod against a wedged nvidia module blocks in D state and cannot be # killed, so it needs the same guard as nvidia-smi -- otherwise the repair # hangs here instead of at the probe. local mod rc for mod in nvidia_uvm nvidia_drm nvidia_modeset nvidia; do if lsmod | grep -q "^${mod} "; then log_info " rmmod $mod" run_guarded 30 rmmod "$mod" >>"$LOG_FILE" 2>&1 rc=$? if [ $rc -eq 124 ] || [ $rc -eq 137 ]; then log_err " rmmod $mod HUNG -- the module is wedged in the kernel." log_err " Only a reboot can clear this." REBOOT_REQUIRED=true break elif [ $rc -ne 0 ]; then log_err " Could not unload $mod (still in use). A reboot is required." REBOOT_REQUIRED=true break fi fi done if [ "$REBOOT_REQUIRED" = false ]; then log_info " modprobe nvidia" # modprobe can equally block probing a GPU that is not answering. run_guarded 60 modprobe nvidia >>"$LOG_FILE" 2>&1 \ || log_warn " modprobe nvidia did not complete cleanly (rc=$?)." run_guarded 60 modprobe nvidia_uvm >>"$LOG_FILE" 2>&1 \ || log_warn " modprobe nvidia_uvm did not complete cleanly (rc=$?)." sleep 3 fi if [ "$docker_was_running" = true ]; then quiet systemctl start containerd quiet systemctl start docker fi } # L3: ask the GPU to reset; failing that, remove and rescan it on the PCI bus. reset_gpu() { log_info "[L3] Attempting GPU reset..." local out rc out=$(nvsmi -r); rc=$? if [ $rc -eq 0 ]; then log_ok " nvidia-smi -r completed." sleep 5 return 0 fi log_warn " nvidia-smi -r did not succeed (rc=$rc). Trying a PCIe remove/rescan..." if [ ${#GPU_BDFS[@]} -eq 0 ]; then log_warn " No GPU PCI address known; skipping remove/rescan." return 1 fi local bdf for bdf in "${GPU_BDFS[@]}"; do if [ -w "/sys/bus/pci/devices/$bdf/remove" ]; then log_info " Removing PCI device $bdf..." # A write to sysfs blocks in the kernel if the device is unresponsive. run_guarded 30 sh -c "echo 1 > /sys/bus/pci/devices/$bdf/remove" >>"$LOG_FILE" 2>&1 \ || log_warn " PCI remove of $bdf did not complete." fi done sleep 5 log_info " Rescanning PCI bus..." run_guarded 60 sh -c 'echo 1 > /sys/bus/pci/rescan' >>"$LOG_FILE" 2>&1 \ || log_warn " PCI rescan did not complete." sleep 10 return 0 } # L4a: rebuild the DKMS module against the running kernel. rebuild_dkms() { log_info "[L4a] Rebuilding NVIDIA kernel modules via DKMS..." command -v dkms >/dev/null 2>&1 || { log_warn " dkms not installed; skipping."; return 1; } quiet apt-get install -y "linux-headers-$(uname -r)" if quiet dkms autoinstall; then log_ok " DKMS rebuild completed." REBOOT_REQUIRED=true return 0 fi log_err " DKMS rebuild failed. See $LOG_FILE." return 1 } # L4b: full purge and pinned reinstall of the known-good driver. # Mirrors monitor_nvidia.sh, but reuses a cached .deb and never calls # nvidia-smi unguarded. reinstall_driver() { log_head "[L4b] Full NVIDIA driver reinstall -> $TARGET_DRIVER_VERSION" if grep -q "^driver_reinstalled=" "$STATE_FILE" 2>/dev/null; then log_warn "A driver reinstall was already performed in a previous run of this script." log_warn "It did not resolve the fault. Not reinstalling again -- suspect hardware." return 1 fi log_info "Clearing any interrupted dpkg state..." quiet dpkg --configure -a log_info "Purging existing NVIDIA/CUDA packages..." quiet apt-get purge -y '*nvidia*' 'cuda*' quiet apt-get autoremove -y rm -rf /usr/local/cuda* /etc/modprobe.d/nvidia-graphics-drivers.conf rm -f /etc/apt/sources.list.d/cuda*.list /etc/apt/preferences.d/cuda-repository-pin-600 rm -rf /var/cuda-repo-ubuntu2204-* # Reuse the cached installer if it is intact -- edge boxes are often # on poor links, and this file is ~3GB. local deb="$TEMP_DIR/$DEB_PACKAGE" if [ -f "$deb" ] && file "$deb" 2>/dev/null | grep -q "Debian binary package"; then log_ok "Reusing cached CUDA package at $deb" else rm -f "$deb" log_info "Downloading CUDA local repo package (this is large; be patient)..." if ! curl -L --retry 3 --retry-delay 5 -o "$deb" "$DEB_PACKAGE_URL" >>"$LOG_FILE" 2>&1; then log_err "Download failed. Check network connectivity to developer.download.nvidia.com." return 1 fi if ! file "$deb" 2>/dev/null | grep -q "Debian binary package"; then log_err "Downloaded file is not a valid .deb package." rm -f "$deb" return 1 fi fi log_info "Setting the CUDA repository pin..." if wget -q "$CUDA_PIN_URL" -O "$TEMP_DIR/cuda-ubuntu2204.pin" >>"$LOG_FILE" 2>&1; then mv "$TEMP_DIR/cuda-ubuntu2204.pin" /etc/apt/preferences.d/cuda-repository-pin-600 else log_warn "Could not fetch the repo pin; continuing without it." fi log_info "Installing the CUDA local repository..." quiet dpkg -i "$deb" cp /var/cuda-repo-ubuntu2204-12-9-local/cuda-*-keyring.gpg /usr/share/keyrings/ 2>>"$LOG_FILE" quiet apt-get update log_info "Installing cuda-drivers..." if ! quiet apt-get install -y cuda-drivers; then log_err "cuda-drivers installation FAILED. See $LOG_FILE." return 1 fi echo "driver_reinstalled=$(date +%s)" >> "$STATE_FILE" log_ok "Driver $TARGET_DRIVER_VERSION installed. A reboot is required to load it." REBOOT_REQUIRED=true return 0 } # Persistently disable GSP firmware. This is the standard workaround for a card # that repeatedly throws Xid 119 (GSP RPC timeout): the driver stops offloading # to the GPU's firmware processor, so the RPC that times out is never made. # Only takes effect once the module is reloaded, i.e. after the next boot. apply_gsp_workaround() { local conf=/etc/modprobe.d/nvidia-gsp.conf log_head "Disabling GSP firmware" if grep -qs 'NVreg_EnableGpuFirmware=0' "$conf"; then log_ok "GSP is already disabled in $conf." else echo "options nvidia NVreg_EnableGpuFirmware=0" > "$conf" log_ok "Wrote $conf" fi if command -v update-initramfs >/dev/null 2>&1; then log_info "Rebuilding initramfs so the option applies at boot..." quiet update-initramfs -u fi REBOOT_REQUIRED=true if [ "$NVSMI_STATE" = "ok" ]; then log_warn "GSP will be disabled from the next boot. The GPU is healthy right now, so" log_warn "an ordinary reboot is enough to apply it." else log_warn "GSP will be disabled from the next boot. Because the GPU firmware is" log_warn "currently wedged, that next boot must be a COLD POWER CYCLE to take effect." COLD_BOOT_REQUIRED=true fi } # Walk the ladder, stopping the moment the GPU answers. repair_ladder() { log_head "PHASE 2: Repair" if [ "$GPU_PRESENT" = false ]; then log_err "The GPU is not visible on the PCI bus. Software cannot fix this." log_err "Power the machine down completely (not a warm reboot), reseat the card" log_err "and its power connectors, then power on." COLD_BOOT_REQUIRED=true return 3 fi if [ "$GPU_OFF_BUS" = true ] || [ "$COLD_BOOT_REQUIRED" = true ]; then if [ "$GSP_FAULT" = true ]; then log_err "The GPU's GSP firmware has stopped responding (Xid 119/120)." log_err "No software repair can reach the card while its firmware is wedged." else log_err "The GPU has fallen off the PCI bus (Xid 79 / rev ff)." fi log_err "This is NOT recoverable by a warm reboot -- a full power cycle is required." return 3 fi if [ "$FORCE_REINSTALL" = true ]; then log_warn "--force-reinstall given; going straight to a full driver reinstall." quiesce_gpu_consumers reinstall_driver return $? fi if [ "$NVSMI_STATE" = "ok" ]; then log_ok "GPU is already responding; no driver repair needed." REPAIRED=true return 0 fi # Everything below unloads or resets the driver. Both are impossible while # the kernel holds references nothing can release -- and ATTEMPTING them is # actively harmful: an rmmod that blocks in D state leaves the module with a # negative refcount and adds one more unkillable task. Refuse instead. if [ "$MODULE_STATE_CORRUPT" = true ]; then log_err "An NVIDIA module has a corrupted (negative) reference count." log_err "It cannot be unloaded, so the repair ladder cannot possibly succeed." log_err "Not attempting it -- that is what corrupted the count in the first place." REBOOT_REQUIRED=true return 2 fi if [ -n "$GPU_DSTATE_PIDS" ]; then log_err "Tasks are wedged inside the NVIDIA driver and cannot be signalled:" log_err " $GPU_DSTATE_PIDS" log_err "They hold references on the nvidia module, so no rmmod and no GPU reset can" log_err "succeed. Skipping the repair ladder rather than wedging more processes." REBOOT_REQUIRED=true return 2 fi # --- L1 + L2 ------------------------------------------------------- quiesce_gpu_consumers reload_nvidia_modules invalidate_nvsmi_probe # the modules changed -- a fresh probe is warranted if [ "$REBOOT_REQUIRED" = false ] && gpu_healthy; then log_ok "GPU recovered after a module reload. Driver version: $NVSMI_DRIVER" REPAIRED=true return 0 fi log_warn "GPU still not responding after a module reload." # --- L3 ------------------------------------------------------------ if [ "$REBOOT_REQUIRED" = false ]; then reset_gpu invalidate_nvsmi_probe # the GPU was reset -- a fresh probe is warranted if gpu_healthy; then log_ok "GPU recovered after a reset. Driver version: $NVSMI_DRIVER" REPAIRED=true return 0 fi log_warn "GPU still not responding after a reset." fi # --- L4 ------------------------------------------------------------ if [ "$DKMS_OK" = false ] || [ "$NVSMI_STATE" = "missing" ] \ || { [ -n "$DISK_MOD_VERSION" ] && [ "$DISK_MOD_VERSION" != "$TARGET_DRIVER_VERSION" ]; }; then rebuild_dkms invalidate_nvsmi_probe # modules were rebuilt -- a fresh probe is warranted if [ "$REBOOT_REQUIRED" = false ] && gpu_healthy; then log_ok "GPU recovered after a DKMS rebuild." REPAIRED=true return 0 fi fi if [ "$REBOOT_REQUIRED" = false ]; then reinstall_driver || return 1 fi return 2 } #======================================================================= # PHASE 3 -- Container runtime restore #======================================================================= restore_container_runtime() { log_head "PHASE 3: Container runtime" if ! command -v docker >/dev/null 2>&1; then log_err "Docker is not installed. Run the bundle's docker-install.sh first." return 1 fi if ! dpkg -l 2>/dev/null | grep -qw nvidia-container-toolkit; then log_warn "nvidia-container-toolkit is missing. Installing..." curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \ | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg 2>>"$LOG_FILE" curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \ | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \ > /etc/apt/sources.list.d/nvidia-container-toolkit.list quiet apt-get update quiet apt-get install -y nvidia-container-toolkit else log_ok "nvidia-container-toolkit is installed." fi if ! run_guarded 30 docker info 2>/dev/null | grep -q nvidia; then log_warn "The NVIDIA runtime is not registered with Docker. Configuring..." quiet nvidia-ctk runtime configure --runtime=docker else log_ok "Docker already exposes the NVIDIA runtime." fi log_info "Restarting the Docker daemon..." quiet systemctl restart docker sleep 5 # Smoke test using an image that is already local where possible -- # a broken edge box may have no working route to a registry. local test_image test_image=$(docker images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null \ | grep -E '^(nvidia/cuda|ubuntu):' | grep -v '' | head -n1) [ -z "$test_image" ] && test_image="nvidia/cuda:12.0.0-base-ubuntu22.04" log_info "Testing GPU access inside a container using $test_image..." if run_guarded 120 docker run --rm --runtime=nvidia --gpus all "$test_image" nvidia-smi >>"$LOG_FILE" 2>&1; then log_ok "Containers can reach the GPU." return 0 fi log_err "GPU is NOT accessible from inside a container." log_err "Check 'nvidia-ctk runtime configure' output and /etc/docker/daemon.json." return 1 } #======================================================================= # PHASE 4 -- Production service restore #======================================================================= restore_services() { log_head "PHASE 4: Production services" cd "$BUNDLE_DIR" || return 1 for mp in "${MOUNT_PATHS[@]}"; do if ! mountpoint -q "$mp"; then log_err "$mp is not mounted. Refusing to start services -- data would be written to the root disk." return 1 fi done log_info "Starting the edge container stack..." if ! run_guarded 600 docker compose up -d >>"$LOG_FILE" 2>&1; then log_err "docker compose up failed. See $LOG_FILE." return 1 fi log_info "Waiting 60s for containers to settle..." sleep 60 # Report anything that is not up. local ps_out bad ps_out=$(docker compose ps 2>/dev/null) echo "$ps_out" >> "$LOG_FILE" bad=$(echo "$ps_out" | tail -n +2 | grep -viE '(running|healthy|[[:space:]]Up[[:space:]])') if [ -n "$bad" ]; then log_warn "Containers not in a running state:" log_warn "$bad" else log_ok "All compose services are running." fi local restarting restarting=$(docker ps --filter 'status=restarting' --format '{{.Names}}' 2>/dev/null) if [ -n "$restarting" ]; then log_err "Containers stuck in a restart loop: $(echo "$restarting" | tr '\n' ' ')" log_err "Inspect with: docker logs --tail 100 " fi # pva_service is the GPU consumer in this stack -- verify it specifically. if grep -q 'pva_service' docker-compose.yml 2>/dev/null; then local pva pva=$(docker ps --filter 'name=pva' --format '{{.Names}} {{.Status}}' 2>/dev/null) if [ -n "$pva" ]; then log_ok "GPU analytics service: $pva" else log_err "pva_service (the GPU consumer) is not running -- analytics will not work." fi fi # The engines directory is populated once analytics initialises properly. if [ -d "./engines" ]; then if [ -n "$(ls -A ./engines 2>/dev/null)" ]; then log_ok "Engine files present in ./engines -- analytics initialised." else log_warn "./engines is still empty. Analytics may take up to 20 minutes to initialise." log_warn "Monitor with: watch -n 30 'ls -A ./engines | wc -l'" fi fi # Companion monitors installed by the original installer. log_info "Checking companion services..." local svc for svc in log-sync.service vsaas_edge_updater.service; do if systemctl list-unit-files 2>/dev/null | grep -q "^${svc}"; then if systemctl is-active --quiet "$svc"; then log_ok " $svc is active." else log_warn " $svc is inactive. Restarting..." quiet systemctl restart "$svc" fi fi done local cron_count cron_count=$(crontab -l 2>/dev/null | grep -cE 'ip_monitor|system_monitor|docker_health_check|freem|edge_sync_logs') if [ "$cron_count" -ge 1 ]; then log_ok " $cron_count monitoring cron job(s) registered." else log_warn " No monitoring cron jobs found. Re-run the bundle's install_*.sh scripts to restore them." fi return 0 } #======================================================================= # Reporting #======================================================================= # Turn the collected findings into a named root cause and a recommended action. assess() { log_head "Assessment" if [ "$GPU_PRESENT" = false ]; then log_err "LIKELY CAUSE: The GPU is not enumerated on the PCI bus at all." log_err "ACTION: Full power cycle, then reseat the card and its PCIe power cables." return fi if [ "$GPU_OFF_BUS" = true ] || [[ " $XID_CODES " == *" 79 "* ]]; then log_err "LIKELY CAUSE: GPU fell off the PCI bus (Xid 79). Typically power delivery," log_err " thermal shutdown, a failing riser, or a dying card." log_err "ACTION: Cold power cycle (mains off ~30s). A warm reboot will NOT clear this." return fi if [ "$MODULE_STATE_CORRUPT" = true ]; then log_err "LIKELY CAUSE: An NVIDIA module's reference count has gone negative, meaning a" log_err " previous unload was interrupted mid-flight. The kernel's view of" log_err " the driver is now inconsistent and cannot be repaired in place." log_err "ACTION: Reboot is mandatory. If D-state tasks are also present the shutdown will" log_err " hang, so use a forced restart or a power cycle." return fi if [ "$GSP_FAULT" = true ]; then log_err "LIKELY CAUSE: The GPU's GSP firmware stopped responding (Xid 119/120). Every" log_err " driver call then blocks in the kernel forever, and the tasks that" log_err " made them become unkillable -- which also stops the box shutting" log_err " down cleanly." log_err "ACTION: Cold power cycle (mains off ~30s). A warm reboot does NOT reset the GSP," log_err " so the fault survives it. To stop this recurring, re-run with" log_err " --disable-gsp, then power cycle." return fi if [[ " $XID_CODES " == *" 48 "* ]] || [[ " $XID_CODES " == *" 94 "* ]]; then log_err "LIKELY CAUSE: Uncorrectable ECC memory error on the GPU -- hardware fault." log_err "ACTION: Plan card replacement. Recovery here is temporary at best." return fi if [ -n "$LOADED_MOD_VERSION" ] && [ -n "$DISK_MOD_VERSION" ] \ && [ "$LOADED_MOD_VERSION" != "$DISK_MOD_VERSION" ]; then log_err "LIKELY CAUSE: Driver upgraded underneath a running kernel." log_err " Resident module $LOADED_MOD_VERSION vs on-disk $DISK_MOD_VERSION." log_err "ACTION: Reboot. Then pin the driver so unattended-upgrades cannot repeat this:" log_err " sudo apt-mark hold cuda-drivers nvidia-driver-\${TARGET_DRIVER_VERSION%%.*}" return fi if [ "$DKMS_OK" = false ]; then log_err "LIKELY CAUSE: Kernel was updated to $(uname -r) but the NVIDIA DKMS module" log_err " was never rebuilt for it, so the driver cannot bind." log_err "ACTION: The repair phase rebuilds it via DKMS, then reboot." return fi if [ "$SECUREBOOT_ON" = true ] && [ "$NVSMI_STATE" != "ok" ]; then log_err "LIKELY CAUSE: Secure Boot is enabled and is refusing the unsigned NVIDIA module." log_err "ACTION: Enrol the MOK key, or disable Secure Boot in firmware." return fi if [ -n "$DSTATE_PROCS" ] && [ "$NVSMI_STATE" = "hung" ]; then log_err "LIKELY CAUSE: Processes are wedged in uninterruptible sleep on the GPU," log_err " which is exactly why nvidia-smi never returns over SSH." log_err "ACTION: They cannot be killed, so no module unload and no graceful shutdown" log_err " can succeed. See the reboot guidance in the summary below." return fi case "$NVSMI_STATE" in hung) log_err "LIKELY CAUSE: Driver/GPU communication wedged with no Xid recorded." log_err "ACTION: Module reload, then GPU reset, then reboot. This script tries all three." ;; missing) log_err "LIKELY CAUSE: NVIDIA driver is not installed (or was purged by an apt operation)." log_err "ACTION: Full pinned reinstall of $TARGET_DRIVER_VERSION, then reboot." ;; error) log_err "LIKELY CAUSE: nvidia-smi runs but cannot talk to the driver." log_err "ACTION: Module reload, then driver reinstall if that fails." ;; ok) log_ok "GPU is responding normally. If the application is still down, the fault is" log_ok "in the container stack rather than the GPU -- check 'docker compose ps'." ;; esac } # Ship the collected diagnostics to the support server. # # Called at the end of final_report(), so a box that is about to be power-cycled # has already handed its evidence over -- on this hardware the logs are often # the only thing that survives the fault. # # The whole transfer runs through run_guarded: an edge box with no route to the # log server must not leave the operator staring at a stalled rsync when the # real instruction is "power cycle now". upload_logs() { if [ "$UPLOAD_LOGS" != true ]; then log_info "Log upload disabled (--no-upload). Everything stays in $LOG_DIR." return 0 fi log_head "Uploading logs to $REMOTE_USER@$REMOTE_HOST:$SSH_PORT" if ! command -v rsync >/dev/null 2>&1; then log_warn "rsync is not installed; trying to install it..." quiet apt-get install -y rsync if ! command -v rsync >/dev/null 2>&1; then log_err "rsync unavailable -- skipping upload. Logs remain in $LOG_DIR." return 1 fi fi # A password file beats the built-in default. Read it, never echo it. if [ -n "$PASS_FILE" ]; then if [ -r "$PASS_FILE" ]; then REMOTE_PASS=$(head -n1 "$PASS_FILE") log_info "Password read from $PASS_FILE" else log_err "--pass-file '$PASS_FILE' is not readable. Skipping upload." return 1 fi fi local host stamp dest host=$(hostname 2>/dev/null | tr -cd '[:alnum:]._-') [ -n "$host" ] || host="unknown-edge" stamp=$(basename "$DIAG_DIR" | sed 's/^diagnostics_//') [ -n "$stamp" ] || stamp="manual" dest="$REMOTE_BASE/$host/gpu_recovery_$stamp" # run.log / cmd_trace.log were already folded into DIAG_DIR by final_report, # so the uploaded tree is self-describing. The tarball is just a compressed # copy of this same tree, so it is deliberately not sent twice. log_info "Destination: $dest" log_info "Payload : $(du -sh "$DIAG_DIR" 2>/dev/null | cut -f1) in $(find "$DIAG_DIR" -type f 2>/dev/null | wc -l) files" local ssh_opts="-p $SSH_PORT -o StrictHostKeyChecking=accept-new" ssh_opts="$ssh_opts -o ConnectTimeout=20 -o ServerAliveInterval=10 -o ServerAliveCountMax=3" local rc=1 if [ -n "$REMOTE_KEY" ] && [ -r "$REMOTE_KEY" ]; then log_info "Auth: SSH key $REMOTE_KEY" run_guarded 900 rsync -az --partial --mkpath \ -e "ssh -i $REMOTE_KEY $ssh_opts" \ "$DIAG_DIR/" "$REMOTE_USER@$REMOTE_HOST:$dest/" >>"$LOG_FILE" 2>&1 rc=$? if [ $rc -ne 0 ]; then # --mkpath needs rsync >= 3.2.3; Ubuntu 22.04 ships 3.2.7 but a # remote on an older release may not. Create the directory instead. log_info "Retrying without --mkpath..." run_guarded 900 rsync -az --partial \ -e "ssh -i $REMOTE_KEY $ssh_opts" \ --rsync-path="mkdir -p '$dest' && rsync" \ "$DIAG_DIR/" "$REMOTE_USER@$REMOTE_HOST:$dest/" >>"$LOG_FILE" 2>&1 rc=$? fi else if ! command -v sshpass >/dev/null 2>&1; then log_warn "sshpass is not installed; trying to install it..." quiet apt-get install -y sshpass fi if ! command -v sshpass >/dev/null 2>&1; then log_err "No usable authentication: sshpass is missing and no --ssh-key was given." log_err "Logs remain in $LOG_DIR. Copy them by hand with:" log_err " rsync -az $DIAG_DIR/ $REMOTE_USER@$REMOTE_HOST:$dest/" return 1 fi log_info "Auth: password" # sshpass -e reads the password from $SSHPASS. Deliberately NOT # `sshpass -p "$REMOTE_PASS"`, which would place the credential in argv: # visible in `ps` to every user on the box, and captured verbatim by # this script's own command-trace log. export SSHPASS="$REMOTE_PASS" run_guarded 900 sshpass -e rsync -az --partial --mkpath \ -e "ssh $ssh_opts" \ "$DIAG_DIR/" "$REMOTE_USER@$REMOTE_HOST:$dest/" >>"$LOG_FILE" 2>&1 rc=$? if [ $rc -ne 0 ]; then log_info "Retrying without --mkpath..." run_guarded 900 sshpass -e rsync -az --partial \ -e "ssh $ssh_opts" \ --rsync-path="mkdir -p '$dest' && rsync" \ "$DIAG_DIR/" "$REMOTE_USER@$REMOTE_HOST:$dest/" >>"$LOG_FILE" 2>&1 rc=$? fi unset SSHPASS fi case $rc in 0) log_ok "Logs uploaded to $REMOTE_HOST:$dest" ;; 124|137) log_err "Upload TIMED OUT -- this box may have no route to $REMOTE_HOST." log_err "Logs remain in $LOG_DIR." ;; *) log_err "Upload FAILED (rsync rc=$rc). The rsync output is in $LOG_FILE." log_err "Logs remain in $LOG_DIR. Retry by hand with:" log_err " rsync -az $DIAG_DIR/ $REMOTE_USER@$REMOTE_HOST:$dest/" ;; esac return $rc } final_report() { local rc=$1 log_head "Recovery summary" probe_nvidia_smi log_info "GPU state : $NVSMI_STATE${NVSMI_DRIVER:+ (driver $NVSMI_DRIVER)}" log_info "Diagnostics saved : $DIAG_DIR" log_info "Full log : $LOG_FILE" case $rc in 0) log_ok "SYSTEM RECOVERED -- GPU is responding and services are up." ;; 10) log_warn "DIAGNOSIS ONLY -- no changes were made." log_warn "Re-run without --diagnose-only to attempt the repair." ;; 2) log_warn "A REBOOT IS REQUIRED to complete recovery." if [ "$SHUTDOWN_WILL_HANG" = true ]; then log_err "Do NOT use plain 'sudo reboot' on this box right now." log_err "Unkillable (D-state) processes are holding the GPU, and systemd waits" log_err "for every process to exit before restarting. It will stall at" log_err "'Reached target Shutdown' and never come back -- which is why a power" log_err "cycle has been the only way to recover it." log_warn "Force the restart past userspace instead:" log_warn " sudo sync && sudo systemctl reboot --force --force" log_warn "or, if even that stalls:" log_warn " sudo sync && echo b | sudo tee /proc/sysrq-trigger" log_warn "(Or just re-run this script with --allow-reboot; it now does this for you.)" else log_warn "Run: sudo reboot" fi log_warn "After it comes back, re-run this script to finish restoring services:" log_warn " sudo bash $0 --bundle-dir $BUNDLE_DIR" ;; 3) if [ "$GSP_FAULT" = true ]; then log_err "GPU FIRMWARE (GSP) FAULT -- Xid 119/120." log_err "Power the machine off completely (mains off ~30 seconds), then power on." log_err "A warm reboot will not clear this: the GSP keeps its state across one," log_err "so the box comes back with the same fault. That matches what you have" log_err "been seeing -- only the power cycle works." log_err "" log_err "To stop it recurring, run the GPU with GSP firmware disabled:" log_err " sudo bash $0 --disable-gsp # then power cycle" log_err "This sets NVreg_EnableGpuFirmware=0, which is the standard workaround" log_err "for repeated Xid 119 on datacentre/RTX cards." else log_err "HARDWARE FAULT SUSPECTED." log_err "Power the machine off completely (pull mains power for ~30 seconds)," log_err "reseat the GPU and its PCIe power cables, then power on." log_err "If nvidia-smi still hangs afterwards, the card needs replacing." fi log_err "Send $DIAG_DIR to support." ;; *) log_err "RECOVERY INCOMPLETE. Review $LOG_FILE and the diagnostics in $DIAG_DIR." ;; esac # Bundle diagnostics for support hand-off. Fold the run log and the command # trace in first so the tarball is self-contained. cp -f "$LOG_FILE" "$DIAG_DIR/run.log" 2>/dev/null cp -f "$TRACE_FILE" "$DIAG_DIR/cmd_trace.log" 2>/dev/null TARBALL="$LOG_DIR/gpu_diagnostics_$(basename "$DIAG_DIR" | sed 's/^diagnostics_//').tar.gz" if tar -czf "$TARBALL" -C "$(dirname "$DIAG_DIR")" "$(basename "$DIAG_DIR")" 2>/dev/null; then log_ok "Support bundle: $TARBALL ($(du -h "$TARBALL" 2>/dev/null | cut -f1))" log_info " $(find "$DIAG_DIR" -type f 2>/dev/null | wc -l) files collected" else log_warn "Could not create the support tarball; the diagnostics tree is still intact." TARBALL="" fi log_info "Logs written this run:" log_info " run log : $LOG_FILE" log_info " cmd trace : $TRACE_FILE" log_info " diagnostics : $DIAG_DIR/" log_info " summary : $DIAG_DIR/00-SUMMARY.txt" log_info " tarball : ${TARBALL:-}" # Hand the evidence over before anything reboots or power-cycles the box. upload_logs } maybe_reboot() { [ "$ALLOW_REBOOT" = true ] || return 0 echo "pending_reboot=$(date +%s)" >> "$STATE_FILE" if [ "$SHUTDOWN_WILL_HANG" = false ]; then log_warn "Rebooting in 15 seconds to complete recovery (--allow-reboot given)..." sleep 15 reboot exit 2 fi # A normal reboot would stall forever waiting on the unkillable tasks, so # skip the userspace teardown entirely. Flush filesystems by hand first, # because nothing else is going to get the chance to. log_warn "Unkillable processes are present, so a graceful reboot would hang." log_warn "Forcing an immediate restart in 15 seconds; syncing filesystems first..." sleep 15 sync; sync # Best effort: remount everything read-only so the forced reset is clean. mount -o remount,ro -a >>"$LOG_FILE" 2>&1 log_warn "Issuing forced restart now." systemctl reboot --force --force >>"$LOG_FILE" 2>&1 sleep 10 # Last resort: SysRq immediate reboot, bypassing systemd altogether. echo 1 > /proc/sys/kernel/sysrq 2>/dev/null echo b > /proc/sysrq-trigger 2>/dev/null exit 2 } #======================================================================= # Main #======================================================================= main() { parse_args "$@" preflight diagnose assess if [ "$DISABLE_GSP" = true ]; then apply_gsp_workaround fi if [ "$DIAGNOSE_ONLY" = true ]; then final_report 10 exit 0 fi if [ "$SERVICES_ONLY" = true ]; then log_warn "--services-only given; skipping GPU repair and bringing the stack up." restore_container_runtime || { final_report 1; exit 1; } restore_services || { final_report 1; exit 1; } final_report 0 exit 0 fi repair_ladder local repair_rc=$? if [ $repair_rc -eq 3 ]; then final_report 3 exit 3 fi if [ $repair_rc -eq 2 ] || [ "$REBOOT_REQUIRED" = true ]; then final_report 2 maybe_reboot exit 2 fi if [ $repair_rc -ne 0 ]; then final_report 1 exit 1 fi if ! restore_container_runtime; then final_report 1 exit 1 fi if [ "$SKIP_SERVICES" = true ]; then log_warn "--skip-services given; not starting the container stack." final_report 0 exit 0 fi if ! restore_services; then final_report 1 exit 1 fi # Recovery succeeded -- clear the reinstall guard so a future fault # is allowed to try a driver reinstall again. rm -f "$STATE_FILE" final_report 0 exit 0 } main "$@"