From 8d6f90220916726ef926ca0a29803ec27ad7699f Mon Sep 17 00:00:00 2001 From: grok-reviewer-andresmgsl Date: Wed, 22 Jul 2026 23:23:33 +0000 Subject: [PATCH] Harden watcher: shared config, dual forge backends, safer discovery - Add scripts/lib/common.sh (ORG/BOT/FORGE_BACKEND, logging, process_running) - discover.sh: dedupe review-requested+assignee PRs; JSON+summary formats - discover.sh: FORGE_BACKEND=forgejo via Forgejo API + stoke token - Fix poll-loop process detection (basename, not install path) - health-check: jq-safe JSON; backend-aware auth (gh or stoke) - install-live: require rsync/jq/tmux; soft-warn missing gh/stoke - self-test.sh offline smoke; restore/poll-once use shared config - Docs: Forgejo table, watcher.env, OPERATIONS troubleshooting --- POLL_INSTRUCTIONS.md | 20 ++- README.md | 66 ++++++++-- config/watcher.env.example | 25 ++-- docs/ARCHITECTURE.md | 8 ++ docs/OPERATIONS.md | 22 +++- scripts/discover.sh | 258 +++++++++++++++++++++++++++++++++++-- scripts/health-check.sh | 124 +++++++++++------- scripts/health-loop.sh | 21 ++- scripts/install-live.sh | 20 +++ scripts/lib/common.sh | 65 ++++++++++ scripts/poll-loop.sh | 17 ++- scripts/poll-once.sh | 52 ++++++-- scripts/restore.sh | 22 +++- scripts/self-test.sh | 79 ++++++++++++ 14 files changed, 675 insertions(+), 124 deletions(-) create mode 100755 scripts/lib/common.sh create mode 100755 scripts/self-test.sh diff --git a/POLL_INSTRUCTIONS.md b/POLL_INSTRUCTIONS.md index 51a98bf..690f445 100644 --- a/POLL_INSTRUCTIONS.md +++ b/POLL_INSTRUCTIONS.md @@ -173,6 +173,22 @@ discover → read all context → decide ## Identity -- Bot: `grok-bot-andresmgsl` -- Org: `heavy-duty` +- Bot: `grok-bot-andresmgsl` (override with `BOT_LOGIN` in `config/watcher.env`) +- Org: `heavy-duty` (override with `ORG`) - Role: assigned reviewer / assignee agent — **not** an unsolicited drive-by bot + +## Forgejo backend (optional) + +Set `FORGE_BACKEND=forgejo` in `config/watcher.env` when the heavy-duty forge is +[Forgejo](https://forgejo.heavyduty.builders) instead of GitHub.com. + +| Task | GitHub | Forgejo | +|------|--------|---------| +| Auth | `gh auth status` | `stoke auth status` / `stoke auth login` | +| Discover | `gh search …` | `FORGE_BACKEND=forgejo ./scripts/discover.sh` | +| PR review | `gh pr review` | `stoke pr review -o OWNER -r REPO -n N --event …` | +| PR comment | `gh issue comment` | `stoke pr comment -o OWNER -r REPO -n N -b …` | +| PR show | `gh pr view` | `stoke pr show -o OWNER -r REPO -n N` | + +Discovery still only returns items where this bot is **assignee** or **requested reviewer**. +The same Verdict / de-dupe / no-drive-by rules apply. State keys stay `owner/repo#pr:N`. diff --git a/README.md b/README.md index ec58fc5..80f3a23 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # heavy-duty-watcher -**Org-wide GitHub review agent** for the [heavy-duty](https://github.com/heavy-duty) organization. +**Org-wide review agent** for the [heavy-duty](https://github.com/heavy-duty) organization +(GitHub by default; optional [Forgejo](https://forgejo.heavyduty.builders) backend via [stoke](https://forgejo.heavyduty.builders/heavy-duty/stoke)). Runs as bot **`grok-bot-andresmgsl`**. Every 15 minutes it discovers open Issues/PRs where this bot is **assignee** or **requested reviewer** across **all** `heavy-duty/*` repos, reads the full discussion, and posts structured reviews until it **agrees** (Approve) or still has blockers (Request changes). @@ -11,11 +12,14 @@ Runs as bot **`grok-bot-andresmgsl`**. Every 15 minutes it discovers open Issues ## Features - **Org-wide scope** — one watcher for every repo under `heavy-duty` +- **Dual forge backends** — `FORGE_BACKEND=github` (default) or `forgejo` - **Strict collaboration rules** — clear Verdict/Status, specific, non-redundant, blockers vs nits - **Stateful de-dupe** — tracks per-item head SHA and last action (`owner/repo#pr:N`) +- **Deduped discovery** — PRs that are both review-requested and assigned appear once - **Dual durability** — Grok durable schedulers **and** local tmux loops - **Reboot recovery** — `scripts/restore.sh` rebuilds tmux + resumes the agent -- **Hourly health watchdog** — verifies poll loop, gh auth, freshness; self-heals +- **Hourly health watchdog** — verifies poll loop, auth, freshness; self-heals +- **Offline self-test** — `scripts/self-test.sh` (`bash -n` + structure checks) --- @@ -23,12 +27,23 @@ Runs as bot **`grok-bot-andresmgsl`**. Every 15 minutes it discovers open Issues ```bash git clone https://github.com/grok-bot-andresmgsl/heavy-duty-watcher.git +# or from Forgejo: https://forgejo.heavyduty.builders/grok-reviewer-andresmgsl/heavy-duty-watcher.git cd heavy-duty-watcher -gh auth status # bot account with org access +gh auth status # bot account with org access (GitHub backend) +./scripts/self-test.sh # optional offline smoke ./scripts/install-live.sh ./scripts/restore.sh # after reboot: always this ``` +### Forgejo backend + +```bash +cp config/watcher.env.example config/watcher.env +# set FORGE_BACKEND=forgejo and BOT_LOGIN= +stoke auth login +FORGE_BACKEND=forgejo ./scripts/discover.sh +``` + Arm durable schedulers from a live Grok session using prompts in [`docs/SCHEDULER_PROMPTS.md`](docs/SCHEDULER_PROMPTS.md). --- @@ -37,22 +52,25 @@ Arm durable schedulers from a live Grok session using prompts in [`docs/SCHEDULE ```text heavy-duty-watcher/ -├── README.md ← you are here +├── README.md ├── POLL_INSTRUCTIONS.md ← authoritative review rules (read every cycle) ├── config/ │ ├── state.template.json │ └── watcher.env.example ├── scripts/ -│ ├── discover.sh ← org-wide actionable list +│ ├── lib/common.sh ← shared env + helpers +│ ├── discover.sh ← org-wide actionable list (github|forgejo) │ ├── poll-once.sh ← one headless poll (Grok) │ ├── poll-loop.sh ← every 15m │ ├── health-check.sh │ ├── health-loop.sh ← every 1h + self-heal │ ├── restore.sh ← reboot recovery (tmux) -│ └── install-live.sh ← deploy under ~/heavy-duty-watcher +│ ├── install-live.sh ← deploy under ~/heavy-duty-watcher +│ └── self-test.sh ← offline smoke ├── docs/ │ ├── ARCHITECTURE.md │ ├── OPERATIONS.md +│ ├── ORG_SCOPE.md │ ├── SCHEDULER_PROMPTS.md │ └── REVIEW_PLAYBOOK.md └── logs/ ← runtime (gitignored) @@ -63,7 +81,7 @@ heavy-duty-watcher/ ## How a poll works ```text -discover (org search) +discover (org search / forgejo API) → for each match read ALL comments + reviews + diff + checks compare head SHA / prior bot comments (state.json) @@ -78,6 +96,35 @@ Full rules: [`POLL_INSTRUCTIONS.md`](POLL_INSTRUCTIONS.md). --- +## Configuration + +Copy `config/watcher.env.example` → `config/watcher.env` (gitignored): + +| Variable | Default | Meaning | +|----------|---------|---------| +| `ORG` | `heavy-duty` | GitHub/Forgejo org | +| `BOT_LOGIN` | `grok-bot-andresmgsl` | Bot account login | +| `FORGE_BACKEND` | `github` | `github` or `forgejo` | +| `FORGEJO_URL` | `https://forgejo.heavyduty.builders` | Forge base URL | +| `POLL_INTERVAL_SEC` | `900` | tmux poll-loop sleep | +| `HEALTH_INTERVAL_SEC` | `3600` | health-loop sleep | +| `STALE_POLL_MINUTES` | `45` | health WARN threshold | + +--- + +## Day-to-day commands + +```bash +./scripts/discover.sh # JSON (default) +FORMAT=summary ./scripts/discover.sh # human table +./scripts/health-check.sh +./scripts/self-test.sh +tail -f logs/poll.log logs/health.log +tmux attach -t heavy-duty-watcher +``` + +--- + ## Documentation map | Doc | Audience | @@ -87,6 +134,7 @@ Full rules: [`POLL_INSTRUCTIONS.md`](POLL_INSTRUCTIONS.md). | [docs/OPERATIONS.md](docs/OPERATIONS.md) | Humans operating the watcher | | [docs/SCHEDULER_PROMPTS.md](docs/SCHEDULER_PROMPTS.md) | Copy-paste durable task prompts | | [docs/REVIEW_PLAYBOOK.md](docs/REVIEW_PLAYBOOK.md) | Review style examples | +| [docs/ORG_SCOPE.md](docs/ORG_SCOPE.md) | Org repo notes | --- @@ -96,8 +144,6 @@ Full rules: [`POLL_INSTRUCTIONS.md`](POLL_INSTRUCTIONS.md). ~/heavy-duty-watcher/scripts/restore.sh ``` -Creates/attaches tmux session `heavy-duty-watcher` with: - | Window | Role | |--------|------| | `grok` | Interactive agent (resume session when possible) | @@ -111,7 +157,7 @@ Creates/attaches tmux session `heavy-duty-watcher` with: - Only comments when **assigned** or **review-requested** - Never merges, closes, force-pushes, or reassigns by default -- Secrets are **not** stored in this repo — use `gh auth` on the host +- Secrets are **not** stored in this repo — use `gh auth` / `stoke auth login` on the host --- diff --git a/config/watcher.env.example b/config/watcher.env.example index 3043d72..1e8098c 100644 --- a/config/watcher.env.example +++ b/config/watcher.env.example @@ -1,8 +1,17 @@ -# Copy to watcher.env (gitignored) for local overrides. -WATCHER_DIR="${WATCHER_DIR:-$HOME/heavy-duty-watcher}" -ORG="${ORG:-heavy-duty}" -BOT_LOGIN="${BOT_LOGIN:-grok-bot-andresmgsl}" -POLL_INTERVAL_SEC="${POLL_INTERVAL_SEC:-900}" -HEALTH_INTERVAL_SEC="${HEALTH_INTERVAL_SEC:-3600}" -STALE_POLL_MINUTES="${STALE_POLL_MINUTES:-45}" -TMUX_SESSION="${TMUX_SESSION:-heavy-duty-watcher}" +# Copy to config/watcher.env (gitignored) for local overrides. +# Values below are defaults applied by scripts/lib/common.sh when unset. + +# WATCHER_DIR=$HOME/heavy-duty-watcher +ORG=heavy-duty +BOT_LOGIN=grok-bot-andresmgsl + +# github (default) | forgejo +FORGE_BACKEND=github +FORGEJO_URL=https://forgejo.heavyduty.builders + +POLL_INTERVAL_SEC=900 +HEALTH_INTERVAL_SEC=3600 +STALE_POLL_MINUTES=45 +TMUX_SESSION=heavy-duty-watcher +# LEGACY_TMUX=rig-watcher +# LIMIT=50 diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a278ecd..950cd22 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -30,9 +30,17 @@ It polls GitHub for Issues/PRs where bot **`grok-bot-andresmgsl`** is: │ ▼ GitHub org:heavy-duty + or Forgejo org (FORGE_BACKEND=forgejo) state.json + logs/ ``` +### Forge backends + +| `FORGE_BACKEND` | Auth | Discovery | Review CLI | +|-----------------|------|-----------|------------| +| `github` (default) | `gh` | `gh search` | `gh pr review` | +| `forgejo` | `stoke auth` | Forgejo API via stoke token | `stoke pr review` | + ### Why two poll paths? | Path | Survives | diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index af0de97..1be9e35 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -5,15 +5,25 @@ ```bash # 1. Clone git clone https://github.com/grok-bot-andresmgsl/heavy-duty-watcher.git +# Forgejo mirror (if present): +# git clone https://forgejo.heavyduty.builders/grok-reviewer-andresmgsl/heavy-duty-watcher.git cd heavy-duty-watcher -# 2. Auth -gh auth status # must be grok-bot-andresmgsl (or your reviewer bot) +# 2. Auth (pick backend) +gh auth status # FORGE_BACKEND=github (default) +# stoke auth login # FORGE_BACKEND=forgejo -# 3. Install live + start tmux loops +# 3. Optional overrides +cp config/watcher.env.example config/watcher.env +# edit ORG / BOT_LOGIN / FORGE_BACKEND as needed + +# 4. Offline smoke +./scripts/self-test.sh + +# 5. Install live + start tmux loops ./scripts/install-live.sh -# 4. In a Grok session: arm durable schedulers (15m poll + 1h health) +# 6. In a Grok session: arm durable schedulers (15m poll + 1h health) # Or restore interactive session: ./scripts/restore.sh ``` @@ -62,8 +72,10 @@ Item keys changed from `pr:N` to `heavy-duty/rig#pr:N`. | Symptom | Check | |---------|--------| -| No reviews happening | `./scripts/discover.sh` empty? Not requested. | +| No reviews happening | `FORMAT=summary ./scripts/discover.sh` empty? Not requested. | | Stale last_poll | `health-check.sh` WARN; is poll-loop alive? | | gh 401 | `gh auth login` | +| stoke 401 | `stoke auth login` (Forgejo backend) | | Duplicate spam | Inspect `state.json` `last_head_sha` / `last_action` | | Scheduler missing | Re-arm via Grok; see `docs/SCHEDULER_PROMPTS.md` | +| `install-live` fails | need `rsync`, `jq`, `tmux` | diff --git a/scripts/discover.sh b/scripts/discover.sh index 8534f85..984fb6d 100755 --- a/scripts/discover.sh +++ b/scripts/discover.sh @@ -1,17 +1,255 @@ #!/usr/bin/env bash # Org-wide discovery of actionable PRs/issues for the bot. +# Supports FORGE_BACKEND=github (default) or forgejo. set -euo pipefail -ORG="${ORG:-heavy-duty}" + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/common.sh +source "${SCRIPT_DIR}/lib/common.sh" + +FORMAT="${FORMAT:-json}" # json | summary LIMIT="${LIMIT:-50}" -echo "=== review-requested PRs (org:${ORG}) ===" -gh search prs --owner "${ORG}" --review-requested=@me --state open \ - --json number,title,url,updatedAt,author,assignees,repository --limit "${LIMIT}" +discover_github() { + require_cmd gh jq -echo "=== assignee PRs (org:${ORG}) ===" -gh search prs --owner "${ORG}" --assignee=@me --state open \ - --json number,title,url,updatedAt,author,assignees,repository --limit "${LIMIT}" + local tmp + tmp="$(mktemp -d)" + # shellcheck disable=SC2064 + trap "rm -rf '${tmp}'" EXIT -echo "=== assignee issues (org:${ORG}) ===" -gh search issues --owner "${ORG}" --assignee=@me --state open \ - --json number,title,url,updatedAt,author,assignees,repository --limit "${LIMIT}" + gh search prs --owner "${ORG}" --review-requested=@me --state open \ + --json number,title,url,updatedAt,author,assignees,repository --limit "${LIMIT}" \ + > "${tmp}/pr_rr.json" 2>/dev/null || echo '[]' > "${tmp}/pr_rr.json" + + gh search prs --owner "${ORG}" --assignee=@me --state open \ + --json number,title,url,updatedAt,author,assignees,repository --limit "${LIMIT}" \ + > "${tmp}/pr_as.json" 2>/dev/null || echo '[]' > "${tmp}/pr_as.json" + + gh search issues --owner "${ORG}" --assignee=@me --state open \ + --json number,title,url,updatedAt,author,assignees,repository --limit "${LIMIT}" \ + > "${tmp}/issues.json" 2>/dev/null || echo '[]' > "${tmp}/issues.json" + + # Normalize empty/null files + for f in pr_rr pr_as issues; do + if [[ ! -s "${tmp}/${f}.json" ]] || ! jq empty "${tmp}/${f}.json" 2>/dev/null; then + echo '[]' > "${tmp}/${f}.json" + fi + done + + jq -n \ + --arg org "${ORG}" \ + --arg bot "${BOT_LOGIN}" \ + --slurpfile rr "${tmp}/pr_rr.json" \ + --slurpfile as "${tmp}/pr_as.json" \ + --slurpfile iss "${tmp}/issues.json" \ + ' + def repo_name: + .repository.nameWithOwner // .repository.full_name // "unknown/unknown"; + def pr_key: + (repo_name) + "#pr:" + (.number|tostring); + + ($rr[0] // []) as $rr | + ($as[0] // []) as $as | + ($iss[0] // []) as $iss | + + # set of assignee PR keys + ([$as[] | pr_key] | unique) as $as_keys | + ([$rr[] | pr_key] | unique) as $rr_keys | + + ( + [ + $rr[] | + . + { + kind: "pr", + role: (if ([pr_key] | inside($as_keys)) then "reviewer+assignee" else "reviewer" end) + } + ] + + [ + $as[] | + select(([pr_key] | inside($rr_keys)) | not) | + . + { kind: "pr", role: "assignee" } + ] + ) as $prs | + + ( + [ $iss[] | . + { kind: "issue", role: "assignee" } ] + ) as $issues | + + { + backend: "github", + org: $org, + bot: $bot, + generated_at: (now | strftime("%Y-%m-%dT%H:%M:%SZ")), + counts: { + prs: ($prs | length), + issues: ($issues | length), + total: (($prs | length) + ($issues | length)) + }, + prs: $prs, + issues: $issues + } + ' +} + +discover_forgejo() { + require_cmd jq curl + + local cfg token url tmp page batch n owner repo full + cfg="${STOKE_CONFIG_FILE:-${XDG_CONFIG_HOME:-$HOME/.config}/stoke/config.json}" + if [[ ! -f "${cfg}" ]]; then + echo "error: stoke config not found at ${cfg} (run: stoke auth login)" >&2 + return 1 + fi + token="$(jq -r '.token // empty' "${cfg}")" + url="$(jq -r '.url // empty' "${cfg}")" + url="${url:-$FORGEJO_URL}" + url="${url%/}" + if [[ -z "${token}" ]]; then + echo "error: no token in ${cfg}" >&2 + return 1 + fi + + tmp="$(mktemp -d)" + # shellcheck disable=SC2064 + trap "rm -rf '${tmp}'" EXIT + + api() { + curl -fsS -H "Authorization: token ${token}" -H "Accept: application/json" "$@" + } + + : > "${tmp}/repos.txt" + page=1 + while (( page <= 40 )); do + batch="$(api "${url}/api/v1/orgs/${ORG}/repos?limit=50&page=${page}" 2>/dev/null || echo '[]')" + n="$(echo "${batch}" | jq 'length')" + (( n == 0 )) && break + echo "${batch}" | jq -r '.[].full_name // empty' >> "${tmp}/repos.txt" + (( n < 50 )) && break + page=$((page + 1)) + done + + : > "${tmp}/prs.ndjson" + : > "${tmp}/issues.ndjson" + + while read -r full; do + [[ -z "${full}" ]] && continue + owner="${full%%/*}" + repo="${full#*/}" + + api "${url}/api/v1/repos/${owner}/${repo}/pulls?state=open&limit=50" 2>/dev/null \ + | jq -c --arg full "${full}" --arg bot "${BOT_LOGIN}" ' + .[]? | + ( + ((.requested_reviewers // []) | map(.login) | index($bot) != null) + ) as $is_rev | + ( + (.assignee.login == $bot) + or (((.assignees // []) | map(.login) | index($bot)) != null) + ) as $is_as | + select($is_rev or $is_as) | + { + kind: "pr", + number, + title, + url: .html_url, + updatedAt: .updated_at, + author: (.user.login // null), + head_sha: (.head.sha // null), + repository: { nameWithOwner: $full }, + role: ( + if $is_rev and $is_as then "reviewer+assignee" + elif $is_rev then "reviewer" + else "assignee" + end + ) + } + ' >> "${tmp}/prs.ndjson" 2>/dev/null || true + + api "${url}/api/v1/repos/${owner}/${repo}/issues?state=open&type=issues&limit=50" 2>/dev/null \ + | jq -c --arg full "${full}" --arg bot "${BOT_LOGIN}" ' + .[]? | + select(.pull_request == null) | + select( + (.assignee.login == $bot) + or (((.assignees // []) | map(.login) | index($bot)) != null) + ) | + { + kind: "issue", + number, + title, + url: .html_url, + updatedAt: .updated_at, + author: (.user.login // null), + repository: { nameWithOwner: $full }, + role: "assignee" + } + ' >> "${tmp}/issues.ndjson" 2>/dev/null || true + done < "${tmp}/repos.txt" + + local prs_json issues_json + if [[ -s "${tmp}/prs.ndjson" ]]; then + prs_json="$(jq -s '.' "${tmp}/prs.ndjson")" + else + prs_json='[]' + fi + if [[ -s "${tmp}/issues.ndjson" ]]; then + issues_json="$(jq -s '.' "${tmp}/issues.ndjson")" + else + issues_json='[]' + fi + + jq -n \ + --arg org "${ORG}" \ + --arg bot "${BOT_LOGIN}" \ + --arg url "${url}" \ + --argjson prs "${prs_json}" \ + --argjson issues "${issues_json}" \ + '{ + backend: "forgejo", + forge_url: $url, + org: $org, + bot: $bot, + generated_at: (now | strftime("%Y-%m-%dT%H:%M:%SZ")), + counts: { + prs: ($prs | length), + issues: ($issues | length), + total: (($prs | length) + ($issues | length)) + }, + prs: $prs, + issues: $issues + }' +} + +print_summary() { + jq -r ' + "backend=\(.backend) org=\(.org) bot=\(.bot)", + "counts: prs=\(.counts.prs) issues=\(.counts.issues) total=\(.counts.total)", + "", + (if (.prs|length)>0 then "PRs:" else empty end), + (.prs[]? | " [\(.role)] \(.repository.nameWithOwner // "unknown")!\(.number) — \(.title)"), + (if (.issues|length)>0 then "Issues:" else empty end), + (.issues[]? | " [\(.role)] \(.repository.nameWithOwner // "unknown")#\(.number) — \(.title)"), + (if .counts.total == 0 then "(no actionable items)" else empty end) + ' +} + +main() { + local payload + case "${FORGE_BACKEND}" in + github) payload="$(discover_github)" ;; + forgejo) payload="$(discover_forgejo)" ;; + *) + echo "error: unknown FORGE_BACKEND=${FORGE_BACKEND} (use github or forgejo)" >&2 + exit 2 + ;; + esac + + case "${FORMAT}" in + json) echo "${payload}" | jq . ;; + summary) echo "${payload}" | print_summary ;; + *) echo "error: FORMAT must be json or summary" >&2; exit 2 ;; + esac +} + +main "$@" diff --git a/scripts/health-check.sh b/scripts/health-check.sh index a9027d0..54a1448 100755 --- a/scripts/health-check.sh +++ b/scripts/health-check.sh @@ -2,18 +2,15 @@ # Health check for the heavy-duty org watcher. Exit 0=ok, 1=degraded, 2=down. set -euo pipefail -WATCHER_DIR="${WATCHER_DIR:-$(cd "$(dirname "$0")/.." && pwd)}" -STATE_FILE="${WATCHER_DIR}/state.json" -LOG_DIR="${WATCHER_DIR}/logs" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/common.sh +source "${SCRIPT_DIR}/lib/common.sh" + HEALTH_LOG="${LOG_DIR}/health.log" -TMUX_SESSION="${TMUX_SESSION:-heavy-duty-watcher}" -# Accept legacy session name during migration -LEGACY_TMUX="${LEGACY_TMUX:-rig-watcher}" -STALE_POLL_MINUTES="${STALE_POLL_MINUTES:-45}" mkdir -p "${LOG_DIR}" NOW_EPOCH="$(date -u +%s)" -NOW_ISO="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +NOW_ISO="$(utc_now)" STATUS="ok" ISSUES=() @@ -21,19 +18,40 @@ ok() { ISSUES+=("OK: $1"); } warn() { STATUS="degraded"; ISSUES+=("WARN: $1"); } fail() { STATUS="down"; ISSUES+=("FAIL: $1"); } -if gh api user --jq .login >/dev/null 2>&1; then - LOGIN="$(gh api user --jq .login 2>/dev/null || echo unknown)" - ok "gh authenticated as ${LOGIN}" -else - fail "gh not authenticated" -fi +# --- auth --- +case "${FORGE_BACKEND}" in + github) + if command -v gh >/dev/null 2>&1 && gh api user --jq .login >/dev/null 2>&1; then + LOGIN="$(gh api user --jq .login 2>/dev/null || echo unknown)" + ok "gh authenticated as ${LOGIN}" + if [[ -n "${BOT_LOGIN}" && "${LOGIN}" != "${BOT_LOGIN}" ]]; then + warn "gh login ${LOGIN} != expected BOT_LOGIN ${BOT_LOGIN}" + fi + else + fail "gh not authenticated" + fi + ;; + forgejo) + if command -v stoke >/dev/null 2>&1 && stoke auth status 2>/dev/null | grep -q 'Login:'; then + LOGIN="$(stoke auth status 2>/dev/null | awk '/^Login:/{print $2; exit}')" + ok "stoke authenticated as ${LOGIN:-unknown}" + else + fail "stoke not authenticated (FORGE_BACKEND=forgejo)" + fi + ;; +esac +# --- tmux --- ACTIVE_TMUX="" -if tmux has-session -t "${TMUX_SESSION}" 2>/dev/null; then - ACTIVE_TMUX="${TMUX_SESSION}" -elif tmux has-session -t "${LEGACY_TMUX}" 2>/dev/null; then - ACTIVE_TMUX="${LEGACY_TMUX}" - warn "using legacy tmux session ${LEGACY_TMUX} (prefer ${TMUX_SESSION})" +if command -v tmux >/dev/null 2>&1; then + if tmux has-session -t "${TMUX_SESSION}" 2>/dev/null; then + ACTIVE_TMUX="${TMUX_SESSION}" + elif tmux has-session -t "${LEGACY_TMUX}" 2>/dev/null; then + ACTIVE_TMUX="${LEGACY_TMUX}" + warn "using legacy tmux session ${LEGACY_TMUX} (prefer ${TMUX_SESSION})" + fi +else + warn "tmux not installed" fi if [[ -n "${ACTIVE_TMUX}" ]]; then @@ -47,56 +65,64 @@ else fail "tmux session missing (run scripts/restore.sh)" fi -if ps -eo args | awk '/poll-loop\.sh/ && !/awk/ {found=1} END{exit !found}'; then +if process_running 'poll-loop\.sh'; then ok "poll-loop.sh process running" else fail "poll-loop.sh process not running" fi +# --- state --- if [[ -f "${STATE_FILE}" ]]; then - LAST_POLL="$(jq -r '.last_poll_at // empty' "${STATE_FILE}" 2>/dev/null || true)" - SCOPE="$(jq -r '.scope // .repo // "unknown"' "${STATE_FILE}" 2>/dev/null || echo unknown)" - ok "scope: ${SCOPE}" - if [[ -n "${LAST_POLL}" ]]; then - if LAST_EPOCH="$(date -u -d "${LAST_POLL}" +%s 2>/dev/null)"; then - AGE_MIN=$(( (NOW_EPOCH - LAST_EPOCH) / 60 )) - if (( AGE_MIN > STALE_POLL_MINUTES )); then - warn "last_poll_at is ${AGE_MIN}m old (${LAST_POLL})" + if command -v jq >/dev/null 2>&1; then + LAST_POLL="$(jq -r '.last_poll_at // empty' "${STATE_FILE}" 2>/dev/null || true)" + SCOPE="$(jq -r '.scope // .repo // "unknown"' "${STATE_FILE}" 2>/dev/null || echo unknown)" + ok "scope: ${SCOPE}" + if [[ -n "${LAST_POLL}" && "${LAST_POLL}" != "null" ]]; then + if LAST_EPOCH="$(date -u -d "${LAST_POLL}" +%s 2>/dev/null)"; then + AGE_MIN=$(( (NOW_EPOCH - LAST_EPOCH) / 60 )) + if (( AGE_MIN > STALE_POLL_MINUTES )); then + warn "last_poll_at is ${AGE_MIN}m old (${LAST_POLL})" + else + ok "last_poll_at fresh (${AGE_MIN}m ago: ${LAST_POLL})" + fi else - ok "last_poll_at fresh (${AGE_MIN}m ago: ${LAST_POLL})" + ok "last_poll_at present (${LAST_POLL})" fi else - ok "last_poll_at present (${LAST_POLL})" + warn "state.json has no last_poll_at" fi + SUMMARY="$(jq -r '.last_poll_summary // "n/a"' "${STATE_FILE}" 2>/dev/null || echo n/a)" + ok "last_poll_summary: ${SUMMARY}" else - warn "state.json has no last_poll_at" + warn "jq missing; cannot parse state.json details" fi - SUMMARY="$(jq -r '.last_poll_summary // "n/a"' "${STATE_FILE}" 2>/dev/null || echo n/a)" - ok "last_poll_summary: ${SUMMARY}" else fail "state.json missing" fi [[ -f "${WATCHER_DIR}/POLL_INSTRUCTIONS.md" ]] && ok "POLL_INSTRUCTIONS.md present" || fail "POLL_INSTRUCTIONS.md missing" +# --- write health JSON via jq (safe escaping) --- RESULT_JSON="${LOG_DIR}/health-latest.json" -{ - echo "{" - echo " \"checked_at\": \"${NOW_ISO}\"," - echo " \"status\": \"${STATUS}\"," - echo " \"issues\": [" - first=1 - for line in "${ISSUES[@]}"; do - esc="${line//\"/\\\"}" - if (( first )); then first=0; else echo ","; fi - printf ' "%s"' "${esc}" - done - echo - echo " ]" - echo "}" -} > "${RESULT_JSON}" +if command -v jq >/dev/null 2>&1; then + jq -n \ + --arg t "${NOW_ISO}" \ + --arg s "${STATUS}" \ + --arg backend "${FORGE_BACKEND}" \ + --arg org "${ORG}" \ + --args \ + '{ + checked_at: $t, + status: $s, + backend: $backend, + org: $org, + issues: $ARGS.positional + }' -- "${ISSUES[@]}" > "${RESULT_JSON}" +else + echo "{\"checked_at\":\"${NOW_ISO}\",\"status\":\"${STATUS}\",\"issues\":[]}" > "${RESULT_JSON}" +fi -echo "[${NOW_ISO}] status=${STATUS} | $(IFS=' ; '; echo "${ISSUES[*]}")" >> "${HEALTH_LOG}" +watcher_log "${HEALTH_LOG}" "status=${STATUS} | $(IFS=' ; '; echo "${ISSUES[*]}")" if command -v jq >/dev/null 2>&1 && [[ -f "${STATE_FILE}" ]]; then tmp="$(mktemp)" diff --git a/scripts/health-loop.sh b/scripts/health-loop.sh index b721e77..318e29e 100755 --- a/scripts/health-loop.sh +++ b/scripts/health-loop.sh @@ -2,15 +2,14 @@ # Hourly health check + self-heal for poll-loop. set -euo pipefail -WATCHER_DIR="${WATCHER_DIR:-$(cd "$(dirname "$0")/.." && pwd)}" -LOG_DIR="${WATCHER_DIR}/logs" -INTERVAL_SEC="${HEALTH_INTERVAL_SEC:-3600}" -TMUX_SESSION="${TMUX_SESSION:-heavy-duty-watcher}" -LEGACY_TMUX="${LEGACY_TMUX:-rig-watcher}" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/common.sh +source "${SCRIPT_DIR}/lib/common.sh" + +INTERVAL_SEC="${HEALTH_INTERVAL_SEC}" mkdir -p "${LOG_DIR}" -echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] health-loop started interval=${INTERVAL_SEC}s" \ - >> "${LOG_DIR}/health.log" +watcher_log "${LOG_DIR}/health.log" "health-loop started interval=${INTERVAL_SEC}s" "${WATCHER_DIR}/scripts/health-check.sh" || true @@ -31,9 +30,8 @@ while true; do rc=$? set -e - if ! ps -eo args | awk '/poll-loop\.sh/ && !/awk/ {found=1} END{exit !found}'; then - echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] health-loop: restarting poll-loop" \ - >> "${LOG_DIR}/health.log" + if ! process_running 'poll-loop\.sh'; then + watcher_log "${LOG_DIR}/health.log" "health-loop: restarting poll-loop" sess="$(session_name)" if [[ -n "${sess}" ]]; then tmux list-windows -t "${sess}" -F '#{window_index} #{window_name}' \ @@ -50,7 +48,6 @@ while true; do fi if (( rc != 0 )); then - echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] health-loop: status non-ok rc=${rc}" \ - >> "${LOG_DIR}/health.log" + watcher_log "${LOG_DIR}/health.log" "health-loop: status non-ok rc=${rc}" fi done diff --git a/scripts/install-live.sh b/scripts/install-live.sh index 748d7ac..675ee18 100755 --- a/scripts/install-live.sh +++ b/scripts/install-live.sh @@ -6,6 +6,24 @@ SRC="$(cd "$(dirname "$0")/.." && pwd)" DEST="${DEST:-$HOME/heavy-duty-watcher}" TMUX_SESSION="${TMUX_SESSION:-heavy-duty-watcher}" +missing=() +for c in bash rsync jq tmux; do + command -v "${c}" >/dev/null 2>&1 || missing+=("${c}") +done +if ((${#missing[@]})); then + echo "error: missing required tools: ${missing[*]}" >&2 + echo "install them and re-run (e.g. apt-get install rsync jq tmux)" >&2 + exit 1 +fi + +# Soft deps depending on backend +if [[ "${FORGE_BACKEND:-github}" == "github" ]] && ! command -v gh >/dev/null 2>&1; then + echo "warn: gh not found (required for FORGE_BACKEND=github)" >&2 +fi +if [[ "${FORGE_BACKEND:-github}" == "forgejo" ]] && ! command -v stoke >/dev/null 2>&1; then + echo "warn: stoke not found (required for FORGE_BACKEND=forgejo)" >&2 +fi + echo "Installing watcher: ${SRC} -> ${DEST}" mkdir -p "${DEST}" # Copy tree but preserve local state/logs if DEST already has them @@ -21,6 +39,8 @@ if [[ ! -f "${DEST}/state.json" ]]; then fi chmod +x "${DEST}/scripts/"*.sh +# lib is sourced, not executed +chmod +x "${DEST}/scripts/lib/"*.sh 2>/dev/null || true # Compatibility symlink for old path if [[ ! -e "$HOME/rig-watcher" ]]; then diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh new file mode 100755 index 0000000..1102e38 --- /dev/null +++ b/scripts/lib/common.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Shared helpers for heavy-duty-watcher scripts. +# shellcheck shell=bash +# Usage: source "$(dirname "$0")/lib/common.sh" (from scripts/*.sh) + +# Resolve repo root from the calling script when WATCHER_DIR is unset. +if [[ -z "${WATCHER_DIR:-}" ]]; then + _COMMON_HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + WATCHER_DIR="$(cd "${_COMMON_HERE}/../.." && pwd)" + unset _COMMON_HERE +fi + +# Optional local overrides (never committed — see .gitignore). +if [[ -f "${WATCHER_DIR}/config/watcher.env" ]]; then + # shellcheck disable=SC1091 + set -a + # shellcheck source=/dev/null + source "${WATCHER_DIR}/config/watcher.env" + set +a +fi + +ORG="${ORG:-heavy-duty}" +BOT_LOGIN="${BOT_LOGIN:-grok-bot-andresmgsl}" +# github | forgejo — discovery/posting backend +FORGE_BACKEND="${FORGE_BACKEND:-github}" +FORGEJO_URL="${FORGEJO_URL:-https://forgejo.heavyduty.builders}" +POLL_INTERVAL_SEC="${POLL_INTERVAL_SEC:-900}" +HEALTH_INTERVAL_SEC="${HEALTH_INTERVAL_SEC:-3600}" +STALE_POLL_MINUTES="${STALE_POLL_MINUTES:-45}" +TMUX_SESSION="${TMUX_SESSION:-heavy-duty-watcher}" +LEGACY_TMUX="${LEGACY_TMUX:-rig-watcher}" +LIMIT="${LIMIT:-50}" + +LOG_DIR="${LOG_DIR:-${WATCHER_DIR}/logs}" +STATE_FILE="${STATE_FILE:-${WATCHER_DIR}/state.json}" + +watcher_log() { + local file="$1" + shift + mkdir -p "$(dirname "${file}")" + echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" >> "${file}" +} + +utc_now() { + date -u +%Y-%m-%dT%H:%M:%SZ +} + +require_cmd() { + local c + for c in "$@"; do + if ! command -v "${c}" >/dev/null 2>&1; then + echo "error: required command not found: ${c}" >&2 + return 1 + fi + done +} + +# True if a process whose args contain the given regex is running (excludes awk). +process_running() { + local pattern="$1" + ps -eo args 2>/dev/null | awk -v p="${pattern}" ' + $0 ~ p && $0 !~ /awk/ { found=1 } + END { exit !found } + ' +} diff --git a/scripts/poll-loop.sh b/scripts/poll-loop.sh index 3ebcf93..3fbfff2 100755 --- a/scripts/poll-loop.sh +++ b/scripts/poll-loop.sh @@ -2,13 +2,15 @@ # Long-running poll loop (default 15 minutes) for tmux. set -euo pipefail -WATCHER_DIR="${WATCHER_DIR:-$(cd "$(dirname "$0")/.." && pwd)}" -LOG_DIR="${WATCHER_DIR}/logs" -INTERVAL_SEC="${POLL_INTERVAL_SEC:-900}" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/common.sh +source "${SCRIPT_DIR}/lib/common.sh" + +INTERVAL_SEC="${POLL_INTERVAL_SEC}" mkdir -p "${LOG_DIR}" -echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] poll-loop started interval=${INTERVAL_SEC}s scope=org:heavy-duty (sleep-first)" \ - >> "${LOG_DIR}/poll.log" +watcher_log "${LOG_DIR}/poll.log" \ + "poll-loop started interval=${INTERVAL_SEC}s scope=org:${ORG} backend=${FORGE_BACKEND} (sleep-first)" if [[ "${POLL_NOW:-0}" == "1" ]]; then "${WATCHER_DIR}/scripts/poll-once.sh" || true @@ -16,8 +18,9 @@ fi while true; do sleep "${INTERVAL_SEC}" - if ps -eo args | awk '/heavy-duty-watcher\/scripts\/poll-once\.sh/ && !/awk/ {found=1} END{exit !found}'; then - echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] poll-loop: previous poll still running, skipping" >> "${LOG_DIR}/poll.log" + # Match by script basename so WATCHER_DIR reinstalls / path moves still work. + if process_running 'poll-once\.sh'; then + watcher_log "${LOG_DIR}/poll.log" "poll-loop: previous poll still running, skipping" else "${WATCHER_DIR}/scripts/poll-once.sh" || true fi diff --git a/scripts/poll-once.sh b/scripts/poll-once.sh index 9f36376..fbf8874 100755 --- a/scripts/poll-once.sh +++ b/scripts/poll-once.sh @@ -2,37 +2,61 @@ # Single headless poll via Grok for the heavy-duty org watcher. set -euo pipefail -WATCHER_DIR="${WATCHER_DIR:-$(cd "$(dirname "$0")/.." && pwd)}" -LOG_DIR="${WATCHER_DIR}/logs" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/common.sh +source "${SCRIPT_DIR}/lib/common.sh" + mkdir -p "${LOG_DIR}" -TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +TS="$(utc_now)" PROMPT_FILE="${WATCHER_DIR}/.poll-prompt.txt" +case "${FORGE_BACKEND}" in + github) + DISCOVERY_HINT="gh search prs/issues --owner ${ORG} (review-requested=@me / assignee=@me)" + POST_HINT="Prefer gh pr review / gh issue comment" + BACKEND_LABEL="GitHub" + ;; + forgejo) + DISCOVERY_HINT="bash ${WATCHER_DIR}/scripts/discover.sh # FORGE_BACKEND=forgejo" + POST_HINT="Prefer stoke pr review / stoke pr comment (Forgejo at ${FORGEJO_URL})" + BACKEND_LABEL="Forgejo" + ;; + *) + echo "error: unknown FORGE_BACKEND=${FORGE_BACKEND}" >&2 + exit 2 + ;; +esac + cat > "${PROMPT_FILE}" <> "${LOG_DIR}/poll.log" +watcher_log "${LOG_DIR}/poll.log" "poll-once.sh starting headless grok (org:${ORG} backend:${FORGE_BACKEND})" + +if ! command -v grok >/dev/null 2>&1; then + watcher_log "${LOG_DIR}/poll.log" "poll-once.sh failed: grok CLI not found" + echo "error: grok CLI not found on PATH" >&2 + exit 127 +fi if grok --always-approve --cwd "${HOME}" --max-turns 80 \ --single "$(cat "${PROMPT_FILE}")" \ >> "${LOG_DIR}/poll-headless.log" 2>&1; then - echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] poll-once.sh finished ok" >> "${LOG_DIR}/poll.log" + watcher_log "${LOG_DIR}/poll.log" "poll-once.sh finished ok" else rc=$? - echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] poll-once.sh failed rc=${rc}" >> "${LOG_DIR}/poll.log" + watcher_log "${LOG_DIR}/poll.log" "poll-once.sh failed rc=${rc}" exit "${rc}" fi diff --git a/scripts/restore.sh b/scripts/restore.sh index 747cb08..ac00281 100755 --- a/scripts/restore.sh +++ b/scripts/restore.sh @@ -6,10 +6,10 @@ # ./scripts/restore.sh --status # health snapshot set -euo pipefail -WATCHER_DIR="${WATCHER_DIR:-$(cd "$(dirname "$0")/.." && pwd)}" -STATE_FILE="${WATCHER_DIR}/state.json" -TMUX_SESSION="${TMUX_SESSION:-heavy-duty-watcher}" -LOG_DIR="${WATCHER_DIR}/logs" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/common.sh +source "${SCRIPT_DIR}/lib/common.sh" + mkdir -p "${LOG_DIR}" session_id() { @@ -27,6 +27,7 @@ session_id() { status() { echo "=== heavy-duty-watcher status ===" echo "dir: ${WATCHER_DIR}" + echo "backend: ${FORGE_BACKEND} org: ${ORG} bot: ${BOT_LOGIN}" if [[ -f "${STATE_FILE}" ]]; then jq '{scope, org, bot_login, session_id, last_poll_at, last_poll_summary, health, items: (.items|keys)}' \ "${STATE_FILE}" 2>/dev/null || cat "${STATE_FILE}" @@ -36,7 +37,14 @@ status() { echo tmux ls 2>/dev/null || echo "(no tmux sessions)" echo - echo "gh user: $(gh api user --jq .login 2>/dev/null || echo 'not authenticated')" + case "${FORGE_BACKEND}" in + github) + echo "gh user: $(gh api user --jq .login 2>/dev/null || echo 'not authenticated')" + ;; + forgejo) + echo "stoke: $(stoke auth status 2>/dev/null | head -n 4 | tr '\n' ' ' || echo 'not authenticated')" + ;; + esac echo "session_id: $(session_id || true)" [[ -f "${LOG_DIR}/poll.log" ]] && { echo; echo "last poll log:"; tail -n 5 "${LOG_DIR}/poll.log"; } [[ -f "${LOG_DIR}/health.log" ]] && { echo; echo "last health log:"; tail -n 5 "${LOG_DIR}/health.log"; } @@ -55,7 +63,7 @@ start_or_attach() { local sid resume_prompt sid="$(session_id)" resume_prompt="$(cat <> "${LOG_DIR}/poll.log" + watcher_log "${LOG_DIR}/poll.log" "restore.sh started ${TMUX_SESSION}" exec tmux attach -t "${TMUX_SESSION}" } diff --git a/scripts/self-test.sh b/scripts/self-test.sh new file mode 100755 index 0000000..09ffcb9 --- /dev/null +++ b/scripts/self-test.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Offline smoke checks for the watcher tree (no network required for syntax). +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "${ROOT}" +fail=0 + +echo "== bash -n ==" +while IFS= read -r -d '' f; do + if bash -n "${f}"; then + echo " ok ${f}" + else + echo " FAIL ${f}" + fail=1 + fi +done < <(find scripts -name '*.sh' -print0 | sort -z) + +echo "== required files ==" +for f in \ + POLL_INSTRUCTIONS.md \ + README.md \ + config/state.template.json \ + config/watcher.env.example \ + scripts/lib/common.sh \ + scripts/discover.sh \ + scripts/poll-once.sh \ + scripts/poll-loop.sh \ + scripts/health-check.sh \ + scripts/health-loop.sh \ + scripts/restore.sh \ + scripts/install-live.sh +do + if [[ -f "${f}" ]]; then + echo " ok ${f}" + else + echo " FAIL missing ${f}" + fail=1 + fi +done + +echo "== source common.sh ==" +# shellcheck source=lib/common.sh +if WATCHER_DIR="${ROOT}" source "${ROOT}/scripts/lib/common.sh" \ + && [[ -n "${ORG:-}" && -n "${BOT_LOGIN:-}" ]]; then + echo " ok ORG=${ORG} BOT_LOGIN=${BOT_LOGIN} FORGE_BACKEND=${FORGE_BACKEND}" +else + echo " FAIL sourcing common.sh" + fail=1 +fi + +echo "== state template is valid JSON ==" +if command -v jq >/dev/null 2>&1; then + if jq empty config/state.template.json; then + echo " ok state.template.json" + else + echo " FAIL state.template.json" + fail=1 + fi +else + echo " skip jq not installed" +fi + +echo "== discover --help-ish (dry: unknown backend should fail cleanly) ==" +if FORGE_BACKEND=not-a-backend FORMAT=json bash scripts/discover.sh 2>/dev/null; then + echo " FAIL expected non-zero for bad backend" + fail=1 +else + echo " ok bad backend rejected" +fi + +if (( fail == 0 )); then + echo + echo "self-test: PASS" + exit 0 +fi +echo +echo "self-test: FAIL" +exit 1