heavy-duty-watcher/scripts/health-loop.sh

#!/usr/bin/env bash
# Hourly health check + self-heal for poll-loop.
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
# shellcheck source=lib/common.sh
source "${SCRIPT_DIR}/lib/common.sh"

INTERVAL_SEC="${HEALTH_INTERVAL_SEC}"
mkdir -p "${LOG_DIR}"

watcher_log "${LOG_DIR}/health.log" "health-loop started interval=${INTERVAL_SEC}s"

"${WATCHER_DIR}/scripts/health-check.sh" || true

session_name() {
  if tmux has-session -t "${TMUX_SESSION}" 2>/dev/null; then
    echo "${TMUX_SESSION}"
  elif tmux has-session -t "${LEGACY_TMUX}" 2>/dev/null; then
    echo "${LEGACY_TMUX}"
  else
    echo ""
  fi
}

while true; do
  sleep "${INTERVAL_SEC}"
  set +e
  "${WATCHER_DIR}/scripts/health-check.sh"
  rc=$?
  set -e

  if ! process_running 'poll-loop\.sh'; then
    watcher_log "${LOG_DIR}/health.log" "health-loop: restarting poll-loop"
    sess="$(session_name)"
    if [[ -n "${sess}" ]]; then
      tmux list-windows -t "${sess}" -F '#{window_index} #{window_name}' \
        | awk '$2=="poll-loop"{print $1}' \
        | while read -r idx; do tmux kill-window -t "${sess}:${idx}" 2>/dev/null || true; done
      tmux new-window -t "${sess}" -n poll-loop -c "${WATCHER_DIR}" \
        "bash ${WATCHER_DIR}/scripts/poll-loop.sh"
    else
      tmux new-session -d -s "${TMUX_SESSION}" -n poll-loop -c "${WATCHER_DIR}" \
        "bash ${WATCHER_DIR}/scripts/poll-loop.sh"
      tmux new-window -t "${TMUX_SESSION}" -n health-loop -c "${WATCHER_DIR}" \
        "bash ${WATCHER_DIR}/scripts/health-loop.sh"
    fi
  fi

  if (( rc != 0 )); then
    watcher_log "${LOG_DIR}/health.log" "health-loop: status non-ok rc=${rc}"
  fi
done