heavy-duty-watcher/scripts/health-check.sh

#!/usr/bin/env bash
# Health check for the heavy-duty org watcher. Exit 0=ok, 1=degraded, 2=down.
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
# shellcheck source=lib/common.sh
source "${SCRIPT_DIR}/lib/common.sh"

HEALTH_LOG="${LOG_DIR}/health.log"
mkdir -p "${LOG_DIR}"

NOW_EPOCH="$(date -u +%s)"
NOW_ISO="$(utc_now)"
STATUS="ok"
ISSUES=()

ok()   { ISSUES+=("OK: $1"); }
warn() { STATUS="degraded"; ISSUES+=("WARN: $1"); }
fail() { STATUS="down"; ISSUES+=("FAIL: $1"); }

# --- auth ---
case "${FORGE_BACKEND}" in
  github)
    if command -v gh >/dev/null 2>&1 && gh api user --jq .login >/dev/null 2>&1; then
      LOGIN="$(gh api user --jq .login 2>/dev/null || echo unknown)"
      ok "gh authenticated as ${LOGIN}"
      if [[ -n "${BOT_LOGIN}" && "${LOGIN}" != "${BOT_LOGIN}" ]]; then
        warn "gh login ${LOGIN} != expected BOT_LOGIN ${BOT_LOGIN}"
      fi
    else
      fail "gh not authenticated"
    fi
    ;;
  forgejo)
    if command -v stoke >/dev/null 2>&1 && stoke auth status 2>/dev/null | grep -q 'Login:'; then
      LOGIN="$(stoke auth status 2>/dev/null | awk '/^Login:/{print $2; exit}')"
      ok "stoke authenticated as ${LOGIN:-unknown}"
    else
      fail "stoke not authenticated (FORGE_BACKEND=forgejo)"
    fi
    ;;
esac

# --- tmux ---
ACTIVE_TMUX=""
if command -v tmux >/dev/null 2>&1; then
  if tmux has-session -t "${TMUX_SESSION}" 2>/dev/null; then
    ACTIVE_TMUX="${TMUX_SESSION}"
  elif tmux has-session -t "${LEGACY_TMUX}" 2>/dev/null; then
    ACTIVE_TMUX="${LEGACY_TMUX}"
    warn "using legacy tmux session ${LEGACY_TMUX} (prefer ${TMUX_SESSION})"
  fi
else
  warn "tmux not installed"
fi

if [[ -n "${ACTIVE_TMUX}" ]]; then
  ok "tmux session ${ACTIVE_TMUX} exists"
  if tmux list-windows -t "${ACTIVE_TMUX}" -F '#{window_name}' 2>/dev/null | grep -Eqx 'poll-loop'; then
    ok "tmux window poll-loop present"
  else
    warn "tmux window poll-loop missing"
  fi
else
  fail "tmux session missing (run scripts/restore.sh)"
fi

if process_running 'poll-loop\.sh'; then
  ok "poll-loop.sh process running"
else
  fail "poll-loop.sh process not running"
fi

# --- state ---
if [[ -f "${STATE_FILE}" ]]; then
  if command -v jq >/dev/null 2>&1; then
    LAST_POLL="$(jq -r '.last_poll_at // empty' "${STATE_FILE}" 2>/dev/null || true)"
    SCOPE="$(jq -r '.scope // .repo // "unknown"' "${STATE_FILE}" 2>/dev/null || echo unknown)"
    ok "scope: ${SCOPE}"
    if [[ -n "${LAST_POLL}" && "${LAST_POLL}" != "null" ]]; then
      if LAST_EPOCH="$(date -u -d "${LAST_POLL}" +%s 2>/dev/null)"; then
        AGE_MIN=$(( (NOW_EPOCH - LAST_EPOCH) / 60 ))
        if (( AGE_MIN > STALE_POLL_MINUTES )); then
          warn "last_poll_at is ${AGE_MIN}m old (${LAST_POLL})"
        else
          ok "last_poll_at fresh (${AGE_MIN}m ago: ${LAST_POLL})"
        fi
      else
        ok "last_poll_at present (${LAST_POLL})"
      fi
    else
      warn "state.json has no last_poll_at"
    fi
    SUMMARY="$(jq -r '.last_poll_summary // "n/a"' "${STATE_FILE}" 2>/dev/null || echo n/a)"
    ok "last_poll_summary: ${SUMMARY}"
  else
    warn "jq missing; cannot parse state.json details"
  fi
else
  fail "state.json missing"
fi

[[ -f "${WATCHER_DIR}/POLL_INSTRUCTIONS.md" ]] && ok "POLL_INSTRUCTIONS.md present" || fail "POLL_INSTRUCTIONS.md missing"

# --- write health JSON via jq (safe escaping) ---
RESULT_JSON="${LOG_DIR}/health-latest.json"
if command -v jq >/dev/null 2>&1; then
  jq -n \
    --arg t "${NOW_ISO}" \
    --arg s "${STATUS}" \
    --arg backend "${FORGE_BACKEND}" \
    --arg org "${ORG}" \
    --args \
    '{
      checked_at: $t,
      status: $s,
      backend: $backend,
      org: $org,
      issues: $ARGS.positional
    }' -- "${ISSUES[@]}" > "${RESULT_JSON}"
else
  echo "{\"checked_at\":\"${NOW_ISO}\",\"status\":\"${STATUS}\",\"issues\":[]}" > "${RESULT_JSON}"
fi

watcher_log "${HEALTH_LOG}" "status=${STATUS} | $(IFS=' ; '; echo "${ISSUES[*]}")"

if command -v jq >/dev/null 2>&1 && [[ -f "${STATE_FILE}" ]]; then
  tmp="$(mktemp)"
  jq --arg t "${NOW_ISO}" --arg s "${STATUS}" \
    '.health = {last_check_at: $t, status: $s}' "${STATE_FILE}" > "${tmp}" \
    && mv "${tmp}" "${STATE_FILE}"
fi

echo "status=${STATUS}"
printf '%s\n' "${ISSUES[@]}"
case "${STATUS}" in
  ok) exit 0 ;;
  degraded) exit 1 ;;
  *) exit 2 ;;
esac