1d00544e86
Polls all heavy-duty/* repos every 15m for assignee/review-requested work, reviews with clear Verdicts until agreement, and includes tmux restore + hourly health self-heal. Fully documented.
56 lines
1.8 KiB
Bash
Executable file
56 lines
1.8 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Hourly health check + self-heal for poll-loop.
|
|
set -euo pipefail
|
|
|
|
WATCHER_DIR="${WATCHER_DIR:-$(cd "$(dirname "$0")/.." && pwd)}"
|
|
LOG_DIR="${WATCHER_DIR}/logs"
|
|
INTERVAL_SEC="${HEALTH_INTERVAL_SEC:-3600}"
|
|
TMUX_SESSION="${TMUX_SESSION:-heavy-duty-watcher}"
|
|
LEGACY_TMUX="${LEGACY_TMUX:-rig-watcher}"
|
|
mkdir -p "${LOG_DIR}"
|
|
|
|
echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] health-loop started interval=${INTERVAL_SEC}s" \
|
|
>> "${LOG_DIR}/health.log"
|
|
|
|
"${WATCHER_DIR}/scripts/health-check.sh" || true
|
|
|
|
session_name() {
|
|
if tmux has-session -t "${TMUX_SESSION}" 2>/dev/null; then
|
|
echo "${TMUX_SESSION}"
|
|
elif tmux has-session -t "${LEGACY_TMUX}" 2>/dev/null; then
|
|
echo "${LEGACY_TMUX}"
|
|
else
|
|
echo ""
|
|
fi
|
|
}
|
|
|
|
while true; do
|
|
sleep "${INTERVAL_SEC}"
|
|
set +e
|
|
"${WATCHER_DIR}/scripts/health-check.sh"
|
|
rc=$?
|
|
set -e
|
|
|
|
if ! ps -eo args | awk '/poll-loop\.sh/ && !/awk/ {found=1} END{exit !found}'; then
|
|
echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] health-loop: restarting poll-loop" \
|
|
>> "${LOG_DIR}/health.log"
|
|
sess="$(session_name)"
|
|
if [[ -n "${sess}" ]]; then
|
|
tmux list-windows -t "${sess}" -F '#{window_index} #{window_name}' \
|
|
| awk '$2=="poll-loop"{print $1}' \
|
|
| while read -r idx; do tmux kill-window -t "${sess}:${idx}" 2>/dev/null || true; done
|
|
tmux new-window -t "${sess}" -n poll-loop -c "${WATCHER_DIR}" \
|
|
"bash ${WATCHER_DIR}/scripts/poll-loop.sh"
|
|
else
|
|
tmux new-session -d -s "${TMUX_SESSION}" -n poll-loop -c "${WATCHER_DIR}" \
|
|
"bash ${WATCHER_DIR}/scripts/poll-loop.sh"
|
|
tmux new-window -t "${TMUX_SESSION}" -n health-loop -c "${WATCHER_DIR}" \
|
|
"bash ${WATCHER_DIR}/scripts/health-loop.sh"
|
|
fi
|
|
fi
|
|
|
|
if (( rc != 0 )); then
|
|
echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] health-loop: status non-ok rc=${rc}" \
|
|
>> "${LOG_DIR}/health.log"
|
|
fi
|
|
done
|