rig/drill/drill.sh

#!/usr/bin/env bash
# drill/drill.sh — rig's release drill: the instrument behind drills/README.md.
#
#   ⚠ DESTRUCTIVE, AND MEANT TO BE. Run it on a THROWAWAY Debian machine you
#     can format. It wipes any installed rig and reinstalls from the pinned
#     ref, hardens sshd, sets the hostname, joins the tailnet, installs box
#     and its Incus stack, installs Coolify and Actions runners (GitHub, and
#     Forgejo when --forgejo-instance is given).
#     Never run it on a machine you care about.
#
#   TS_AUTHKEY=tskey-... bash drill/drill.sh \
#     --rig-ref release/0.4.0 --box-ref 0.9.0 \
#     --users ./drill-users --run-id drill-2026-07-24-a \
#     --coolify-version 4.1.2 --runner-repo you/rig \
#     --forgejo-instance https://forgejo.example.com \
#     --forgejo-runner-repo you/drill-probe --yes
#   (--box-ref is a tag: since #103 the box that ships is the BOX_RELEASE tag.)
# rig's drill asserts CONVERGENCE — a machine reaches its role, idempotently.
# The legs (drills/README.md, issue #105):
#
#   1. convergence + idempotence — `rig bootstrap <role> --users <path>`
#      reaches the declared role; a re-run produces an EMPTY state diff,
#      mechanically, never by eye. Rides along: the --host yes assertions
#      (the pinned box installed, its host stack stands — and it STOPS there;
#      the isolation boundary is box's drill's assertion, not this one's).
#   2. db — the real dump/restore round-trip, test/db-integration.sh.
#   3. runner lifecycle — register, take a job, deregister, against a fork.
#      Runs once per forge: `rig runner` against GitHub (--runner-repo), and
#      `rig forgejo-runner` against a Forgejo instance (--forgejo-instance +
#      --forgejo-runner-repo). Both forges ship, so both need evidence; each
#      skips loudly and separately when its inputs are absent (#129).
#   4. coolify install — at a pinned version, AUTOUPDATE=false.
#
# Execution order is 1, 4, 2, 3 — coolify's installer is what puts Docker on
# the box, and leg 2 needs a daemon; running db before coolify would skip a
# leg this same run makes runnable. The record lists legs as they ran.
#
# Exit 0 = no check failed. A FAILED drill still emits a complete record —
# the gate wants evidence, not success — and skipped legs are counted and
# named, never folded into the passes (heavy-duty/box#153's defect class).
#
# The file is one long 'probe && ok "…" || no "…"'. ok/no always return 0, so
# the C-may-run-when-A-is-true trap SC2015 warns about cannot fire here.
# shellcheck disable=SC2015
#
# NOT -e: a failing check is data, not a crash — a drill that aborts on its
# first failure reports one problem per afternoon. NOT pipefail: checks of the
# 'refusal 2>&1 | grep -q text' shape have a left side that exits non-zero BY
# DESIGN, and 'grep -q' SIGPIPEs the left side on early match — box's first
# live run turned both into false FAILs. The pipeline verdict must be grep's
# alone. (box drill/drill.sh's header, the discipline #105 prescribes.)
set -u

SELF="$(readlink -f "$0")"
ROOT="$(cd "$(dirname "$SELF")/.." && pwd)"

REPO="${RIG_REPO:-heavy-duty/rig}"
REF="${RIG_REF:-}"
BOXREPO="${BOX_REPO:-heavy-duty/box}"
BOXREF="${BOX_REF:-}"
# The template registry the converge will read (#110). No explicitness
# demand here, unlike the two refs above: the DEFAULT is already a pin — the
# candidate tree's RIG_TEMPLATES_PIN, read after install from what actually
# landed — so an unset override means "the ref the release will really use",
# not "whatever main was that afternoon".
TPLREPO="${RIG_TEMPLATES_REPO:-heavy-duty/rig-templates}"
TPLREF="${RIG_TEMPLATES_REF:-}"
TPL_SHA=""
TPL_SOURCE="fetched"
ROLE=staging-server
USERS_FILE="${DRILL_USERS_FILE:-}"
RUN_ID="${DRILL_RUN_ID:-drill-$(date -u +%F)}"
RECORD="${DRILL_RECORD:-}"
COOLIFY_VERSION="${DRILL_COOLIFY_VERSION:-}"
RUNNER_REPO="${DRILL_RUNNER_REPO:-}"
RUNNER_WORKFLOW="${DRILL_RUNNER_WORKFLOW:-drill.yml}"
FJ_INSTANCE="${DRILL_FORGEJO_INSTANCE:-}"
FJ_RUNNER_REPO="${DRILL_FORGEJO_RUNNER_REPO:-}"
# The branch the dispatch names. Forgejo's dispatch endpoint requires a ref in
# the body — unlike GitHub's, which defaults to the repo's default branch.
FJ_REF="${DRILL_FORGEJO_REF:-main}"
YES=0

while [ $# -gt 0 ]; do
  case "$1" in
    --yes|-y) YES=1; shift ;;
    --rig-repo) REPO="$2"; shift 2 ;;
    --rig-ref) REF="$2"; shift 2 ;;
    --box-repo) BOXREPO="$2"; shift 2 ;;
    --box-ref) BOXREF="$2"; shift 2 ;;
    --role) ROLE="$2"; shift 2 ;;
    --users) USERS_FILE="$2"; shift 2 ;;
    --run-id) RUN_ID="$2"; shift 2 ;;
    --record) RECORD="$2"; shift 2 ;;
    --coolify-version) COOLIFY_VERSION="$2"; shift 2 ;;
    --runner-repo) RUNNER_REPO="$2"; shift 2 ;;
    --runner-workflow) RUNNER_WORKFLOW="$2"; shift 2 ;;
    --forgejo-instance) FJ_INSTANCE="$2"; shift 2 ;;
    --forgejo-runner-repo) FJ_RUNNER_REPO="$2"; shift 2 ;;
    --forgejo-ref) FJ_REF="$2"; shift 2 ;;
    -h|--help) sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
    *) echo "drill: unknown option: $1 (see --help)" >&2; exit 2 ;;
  esac
done

# --- the reporting verbs (box drill/drill.sh:52-58, the parts worth copying) --
# ok/no/skip/note always return 0: the body stays one long sequence of
# 'probe && ok || no' without fighting the shell. SKIP is its own verb and its
# own counter — a leg that did not run must be visually and arithmetically
# distinct from one that passed (box#153's defect class: a silent skip reads
# as a pass in the record, months later).
pass=0; fail=0; skipped=0; findings=()
ok()   { printf '  \033[32mPASS\033[0m  %s\n' "$*"; pass=$((pass + 1)); }
no()   { printf '  \033[31mFAIL\033[0m  %s\n' "$*"; fail=$((fail + 1)); findings+=("FAIL: $*"); }
skip() { printf '  \033[35mSKIP\033[0m  %s\n' "$*"; skipped=$((skipped + 1)); findings+=("SKIP: $*"); }
note() { printf '  \033[33mNOTE\033[0m  %s\n' "$*"; findings+=("NOTE: $*"); }
inf()  { printf '        %s\n' "$*"; }
phase(){ printf '\n\033[1m══ %s\033[0m\n' "$*"; }

# The record's leg table, appended as legs run. One row per leg, result text
# written at the moment the leg's verdict is known — never reconstructed from
# memory at the end (an invented number is worse than no number).
LEG_NAMES=(); LEG_RESULTS=()
leg() { LEG_NAMES+=("$1"); LEG_RESULTS+=("$2"); }

# forgejo_run_verdict <pre_id> <tasks-json> — the verdict for OUR dispatch:
# success | failed | pending. Reads GET /repos/{o}/{r}/actions/tasks, whose
# shape is NOT GitHub's and was measured against forgejo.heavyduty.builders
# (8.0.3+gitea-1.22.0) on 2026-07-30 rather than read from the docs:
#
#   * There is no `conclusion` field. `status` carries the terminal outcome
#     directly ("success"), where GitHub splits status:completed +
#     conclusion:success. Reading `conclusion` here gets an empty string on
#     every run, which would grade a green job as failed.
#   * `id` is a GLOBAL task id; the run's own URL ends in `run_number`. The
#     pre-dispatch guard therefore compares `id`, exactly as the GitHub leg
#     compares databaseId — an old run must never be read as this one.
#   * The payload lists ASSIGNED tasks only. A run sitting queued is simply
#     absent (measured: 200s of total_count 0 while the web UI showed the run
#     as "job is not started"). So "no new id" is the ONLY signal that the
#     runner never took the job — which is the verdict this leg exists for.
#   * A task appears when ASSIGNED, so it can be seen mid-flight. A
#     non-terminal status is pending, not failed; grading a running job as a
#     failure would make the leg flaky inside its own watch window.
#
# EVERY entry is inspected, and the NEWEST id above pre_id decides — never
# entry[0]. actions/tasks accumulates, so the moment a repo is drilled twice
# our run shares the payload with older ones, and nothing documents the sort
# order. Reading the first entry made a green job report as a timeout, a false
# FAILURE on the gate this leg exists to provide (grok/kimi on !130).
#
# grep-and-sed, not jq: a throwaway drill machine has neither jq nor an
# authenticated forge CLI, the same constraint json_field() carries in
# commands/lib/runner-config.sh. `id` is a bare number, which json_field's
# quoted-value shape cannot read, so this reads both forms itself. Newlines are
# stripped first so a pretty-printed payload parses identically to a compact
# one — the instance documents neither.
forgejo_run_verdict() {
  local pre="$1" file="$2" obj id best_id="" best_st=""
  [ -r "$file" ] || { echo pending; return 0; }
  while IFS= read -r obj; do
    [ -n "$obj" ] || continue
    id="$(printf '%s' "$obj" | grep -o '"id"[[:space:]]*:[[:space:]]*[0-9][0-9]*' \
          | head -n1 | sed 's/.*:[[:space:]]*//')"
    [ -n "$id" ] || continue
    # Strictly newer than the pre-dispatch id. Equal is the run that was
    # already there; lower is older still.
    if [ -n "$pre" ]; then
      [ "$id" -gt "$pre" ] 2>/dev/null || continue
    fi
    if [ -z "$best_id" ] || [ "$id" -gt "$best_id" ] 2>/dev/null; then
      best_id="$id"
      best_st="$(printf '%s' "$obj" | grep -o '"status"[[:space:]]*:[[:space:]]*"[^"]*"' \
                 | head -n1 | sed 's/.*:[[:space:]]*"//; s/"$//')"
    fi
  done <<EOF
$(tr -d '\n' < "$file" 2>/dev/null | grep -o '{[^{}]*}')
EOF
  [ -n "$best_id" ] || { echo pending; return 0; }
  case "$best_st" in
    success)                                  echo success ;;
    failure | cancelled | skipped | timedout) echo failed ;;
    *)                                        echo pending ;;
  esac
}

# forgejo_leg_row <install_ok> <status_ok> <took> <remove_ok> <absent_ok>
# The record row for the Forgejo runner leg. PASS requires the WHOLE lifecycle,
# not just the take-a-job outcome.
#
# Keying the row on <took> alone let it read "PASS — registered, took a job,
# removed" when install had failed, because the dispatched job only needs
# SOMETHING answering runs-on: drill — and this leg removes locally, telling the
# operator to delete the stale runner by hand, so a leftover drill-labeled
# runner from the previous drill is the designed-for aftermath rather than a
# contrived case (codex/grok/kimi on !130). drills/<v>.md is the release's
# durable evidence; a row claiming a lifecycle that did not happen is exactly
# what the gate exists to refuse.
#
# The drill's exit code was never wrong here — every one of those failures also
# called `no`. What was wrong is the row, and the row is what outlives the run.
forgejo_leg_row() {
  local install_ok="$1" status_ok="$2" took="$3" remove_ok="$4" absent_ok="$5"
  if [ "$install_ok" != 1 ] || [ "$status_ok" != 1 ] \
     || [ "$remove_ok" != 1 ] || [ "$absent_ok" != 1 ]; then
    echo "FAIL — see Failed below"
    return 0
  fi
  case "$took" in
    success) echo "PASS — registered, took a job, removed (stale row needs deleting by hand)" ;;
    none)    echo "PARTIAL — registered and removed; took a job: not attempted (no FORGEJO_API_TOKEN)" ;;
    *)       echo "FAIL — see Failed below" ;;
  esac
}

# forgejo_max_task_id <tasks-json> — the highest numeric task id in the payload,
# empty when there is none. This is the PRE-DISPATCH baseline, and it must fold
# max exactly as forgejo_run_verdict does: taking the first id instead names an
# OLD run as the baseline whenever the payload is not newest-first (the order is
# undocumented). A later poll that finds the same body then reads the PREVIOUS
# drill's run as this dispatch's result — a false PASS on the take-a-job
# assertion, which is worse than the false FAIL the same mistake caused inside
# the verdict (grok/kimi, !130). The GitHub leg is safe from this only because
# `gh run list --limit 1` contracts newest-first; this API contracts nothing.
forgejo_max_task_id() {
  local file="$1" id best=""
  [ -r "$file" ] || return 0
  while IFS= read -r id; do
    [ -n "$id" ] || continue
    if [ -z "$best" ] || [ "$id" -gt "$best" ] 2>/dev/null; then best="$id"; fi
  done <<EOF
$(tr -d '\n' < "$file" 2>/dev/null \
  | grep -o '"id"[[:space:]]*:[[:space:]]*[0-9][0-9]*' | sed 's/.*:[[:space:]]*//')
EOF
  printf '%s\n' "$best"
}

# forgejo_token_verdict <resolved_token> <api_token> — ok | mint-failed | no-source.
# #129's acceptance: "Token source present but the instance is unreachable ->
# the leg FAILS; it must not skip and must not pass". A mint that yields
# nothing — unreachable instance, under-scoped token, wrong repo — is a
# CONFIGURED leg failing, and reporting it as "no token source" both writes
# SKIPPED where the record owes a FAIL and sends the operator to check an env
# var they already set. Absent inputs are the only honest skip.
forgejo_token_verdict() {
  if [ -n "$1" ]; then echo ok
  elif [ -n "$2" ]; then echo mint-failed
  else echo no-source
  fi
}


# run_logged <log> <cmd...> — run a long command with its narration in a file
# and a dot every 5s on the terminal: a silent multi-minute apt/install run is
# indistinguishable from a wedge, and that ambiguity has cost box whole
# evenings. Returns the command's exit code.
run_logged() {
  local log="$1"; shift
  inf "watch it live in another terminal:  tail -f $log"
  "$@" >"$log" 2>&1 </dev/null &
  local pid=$!
  while kill -0 "$pid" 2>/dev/null; do printf '.'; sleep 5; done
  printf '\n'
  wait "$pid"
}

# tree_of <cli-path> — the versioned install tree a CLI's symlink chain lands
# in. Both rig and box install as <root>/versions/<v>/bin/<cli> behind a
# 'current' link, so the tree is two dirnames above the resolved binary —
# derived from the chain itself, never from a hardcoded install root (root vs
# user installs put the root in different places).
tree_of() {
  local real
  real="$(readlink -f "$1" 2>/dev/null)"
  # -e as well as -n: GNU readlink -f resolves a path whose LAST component
  # does not exist (exit 0), so a dangling link would hand back a tree that
  # is not there.
  { [ -n "$real" ] && [ -e "$real" ]; } || return 1
  dirname "$(dirname "$real")"
}

# assert_installed_from <what> <tree> <want> — ASSERT WHAT LANDED, never trust
# that the install obeyed. An installer invoked with stale env vars silently
# falls back to its defaults — sane ones since rig#103 landed (box: the
# BOX_RELEASE pin, rig: the latest release), which is what makes the fallback
# invisible — and a drill that thinks it exercised release/X but actually got
# whatever the defaults resolve to has proven nothing about the combination
# that ships — worse than one that fails, because the record it leaves LOOKS
# like evidence. Refusal names both refs, per #105's acceptance criteria.
assert_installed_from() {
  local what="$1" tree="$2" want="$3" got
  got="$(cat "$tree/INSTALLED_FROM" 2>/dev/null || echo '<unreadable>')"
  if [ "$got" != "$want" ]; then
    printf 'drill: FATAL — asked to install %s from %s, but the installed tree says %s.\n' "$what" "$want" "$got" >&2
    printf '  (tree: %s)\n' "$tree" >&2
    printf '  A drill that silently drills the wrong code is worse than one that fails:\n' >&2
    printf '  every result below would describe a tree that is not the candidate. Check\n' >&2
    printf '  the env this drill inherited (a stale RIG_REF/BOX_REF export), fix the\n' >&2
    printf '  pin, and re-run.\n' >&2
    return 1
  fi
  return 0
}

# classify_leg <rc> <outfile> — pass | skip | fail. The skip contract is
# test/db-integration.sh's, copied carefully: it skips CLEANLY (exit 0) with a
# 'skip: <reason>' line when it cannot run, so exit code alone reads a
# not-run leg as a pass. The reason line is the verdict's tiebreaker; a
# non-zero exit is a fail whatever the output says (a die after a skip line
# would be a broken harness, not a skip).
classify_leg() {
  local rc="$1" out="$2"
  if [ "$rc" -eq 0 ] && grep -q '^skip:' "$out" 2>/dev/null; then
    printf 'skip'
  elif [ "$rc" -eq 0 ]; then
    printf 'pass'
  else
    printf 'fail'
  fi
}

# capture_state <outfile> — the convergent surface bootstrap owns, as one
# diffable text file. Leg 1's idempotence claim is decided by capturing this
# BEFORE and AFTER the re-run and diffing — mechanically, because idempotence
# is the single easiest property to convince yourself of by eye (#105).
#
# What is captured is what bootstrap CONVERGES, nothing that legitimately
# moves between two back-to-back runs: no package lists (unattended-upgrades
# may act between captures), no clocks. The manifest is included WHOLE on
# purpose — lib/manifest.sh's contract is that a same-version re-run renders
# byte-identical content (converged_at tracks the version, not the run), so
# the diff ENFORCES that contract instead of exempting it.
#
# Every path is overridable so test/drill.sh proves the capture-and-diff
# machinery against fixtures, without root (repo precedent: RIG_ROLE_MARKER,
# RIG_MANIFEST). Absent files and commands degrade to a deterministic
# '(absent)' — a capture must never fail, only describe.
capture_state() {
  local out="$1" marker manifest ledger autoup hosts u state home keys
  marker="${RIG_ROLE_MARKER:-/etc/rig/role}"
  manifest="${RIG_MANIFEST:-/etc/rig/manifest}"
  ledger="${DRILL_LEDGER:-/etc/rig/users}"
  autoup="${DRILL_AUTOUPGRADES:-/etc/apt/apt.conf.d/20auto-upgrades}"
  hosts="${DRILL_ETC_HOSTS:-/etc/hosts}"
  {
    printf 'hostname: %s\n' "$(hostname 2>/dev/null || echo '(absent)')"
    printf 'hosts.127.0.1.1: %s\n' "$(grep -E '^127\.0\.1\.1[[:space:]]' "$hosts" 2>/dev/null || echo '(absent)')"
    printf 'role-marker: %s\n' "$(cat "$marker" 2>/dev/null || echo '(absent)')"
    printf 'manifest:\n'
    sed 's/^/  /' "$manifest" 2>/dev/null || printf '  (absent)\n'
    printf 'auto-upgrades:\n'
    sed 's/^/  /' "$autoup" 2>/dev/null || printf '  (absent)\n'
    printf 'sshd-effective:\n'
    if command -v sshd >/dev/null 2>&1; then
      sshd -T 2>/dev/null | sort | sed 's/^/  /' || printf '  (sshd -T failed)\n'
    else
      printf '  (sshd absent)\n'
    fi
    # Self's Tags is the FIRST occurrence in the status JSON (Self serializes
    # before Peer). Tags only, nothing livelier: peers joining, IPs renewing
    # or a backend-state flap between two captures is not a convergence diff
    # on this box, and a capture that can move on its own poisons the
    # idempotence verdict with noise.
    printf 'tailscale.self.tags: %s\n' "$(tailscale status --json 2>/dev/null | tr -d '\n ' | grep -o '"Tags":\[[^]]*\]' | head -n1 || true)"
    printf 'users-ledger:\n'
    sed 's/^/  /' "$ledger" 2>/dev/null || printf '  (absent)\n'
    # Per-operator effective state: the account, its groups, its lock state,
    # its keys. sha256 of authorized_keys, not the keys themselves — the
    # capture may end up quoted in a record and keys are long, not secret.
    while read -r u state; do
      [ -n "${u:-}" ] || continue
      if ! id -u "$u" >/dev/null 2>&1; then
        printf 'user.%s: (no account)\n' "$u"
        continue
      fi
      printf 'user.%s: state=%s groups=%s lock=%s\n' "$u" "${state:-active}" \
        "$(id -Gn "$u" 2>/dev/null | tr ' ' ',')" \
        "$(passwd -S "$u" 2>/dev/null | awk '{print $2}' || echo '?')"
      home="$(getent passwd "$u" | cut -d: -f6)"
      keys="$home/.ssh/authorized_keys"
      printf 'user.%s.authorized_keys: %s\n' "$u" \
        "$(sha256sum "$keys" 2>/dev/null | cut -d' ' -f1 || echo '(none)')"
    done < <(cat "$ledger" 2>/dev/null)
    printf 'sudoers.d:\n'
    find "${DRILL_SUDOERS_DIR:-/etc/sudoers.d}" -maxdepth 1 -type f 2>/dev/null | sort \
      | while read -r u; do printf '  %s %s\n' "$(sha256sum "$u" | cut -d' ' -f1)" "$u"; done
    printf 'box: %s\n' "$(command -v box 2>/dev/null || echo '(absent)')"
  } > "$out"
}

# ref_sha <owner/repo> <ref> — the commit the record cites. Tags outrank
# branches (the installer's own precedence, install.sh:117-120). Resolved once
# up front and reused, so the record and the install describe the same instant
# even if the branch moves mid-drill. Empty on failure; the record then says
# 'unresolved' rather than inventing one.
ref_sha() {
  local sha
  sha="$(git ls-remote "https://github.com/$1" "refs/tags/$2" 2>/dev/null | head -n1 | cut -f1)"
  [ -n "$sha" ] || sha="$(git ls-remote "https://github.com/$1" "refs/heads/$2" 2>/dev/null | head -n1 | cut -f1)"
  printf '%s' "${sha:0:7}"
}

# emit_record <path> — drills/<version>.md, in the shape drills/README.md
# defines: what ran, on what host, the pinned refs and SHAs, the numbers, and
# what failed. Emitted on EVERY completed run — a failed drill is still a
# valid record; the gate wants evidence, not success. Skipped legs are listed
# by name: a record with no failures listed reads as "nothing broke", so a leg
# that was not run says so instead of being omitted.
emit_record() {
  local out="$1" i os cpus ram virt line
  os="$(. /etc/os-release 2>/dev/null && printf '%s' "${PRETTY_NAME:-unknown}")"
  cpus="$(nproc 2>/dev/null || echo '?')"
  ram="$(awk '/MemTotal/{printf "%.0f", $2/1024/1024}' /proc/meminfo 2>/dev/null || echo '?')"
  virt="$(systemd-detect-virt 2>/dev/null || echo unknown)"
  {
    printf '# Release drill — %s — %s\n\n' "$DRILL_VERSION" "$(date -u +%F)"
    printf 'Run ID: %s. Host: %s, %s vCPU / %s GB RAM (%s).\n' "$RUN_ID" "${os:-unknown}" "$cpus" "$ram" "$virt"
    printf 'Candidate refs: rig@%s (RIG_REF=%s), box@%s (BOX_REF=%s).\n' \
      "${RIG_SHA:-unresolved}" "$REF" "${BOX_SHA:-unresolved}" "$BOXREF"
    printf 'Template registry: %s@%s (ref %s, %s) — the rig-templates source the converge read (#110/#153).\n' \
      "${TPLREPO:-heavy-duty/rig-templates}" "${TPL_SHA:-unresolved}" \
      "${TPLREF:-unresolved}" "${TPL_SOURCE:-fetched}"
    printf 'Instrument: drill/drill.sh, legs in execution order.\n\n'
    printf '| Leg | Result |\n'
    printf '| --- | --- |\n'
    for i in "${!LEG_NAMES[@]}"; do
      printf '| %s | %s |\n' "${LEG_NAMES[$i]}" "${LEG_RESULTS[$i]}"
    done
    printf '\nChecks: %s passed, %s failed, %s skipped.\n' "$pass" "$fail" "$skipped"
    if [ "$fail" -eq 0 ] && [ "$skipped" -eq 0 ]; then
      printf '\nFailed: nothing. Every leg ran and every check passed.\n'
    else
      # printf --: a format opening with '- ' reads as an option to bash's
      # printf and emits NOTHING — a record whose Failed section silently
      # vanished is exactly the lie this file exists to make impossible.
      [ "$fail" -gt 0 ] && printf '\nFailed:\n'
      for line in "${findings[@]:-}"; do
        case "$line" in FAIL:*) printf -- '- %s\n' "$line" ;; esac
      done
      [ "$skipped" -gt 0 ] && printf '\nSkipped — these did NOT run, and this record is not evidence for them:\n'
      for line in "${findings[@]:-}"; do
        case "$line" in SKIP:*) printf -- '- %s\n' "$line" ;; esac
      done
    fi
    printf '\nThe isolation boundary was NOT asserted here: it is box'\''s drill'\''s\n'
    printf 'assertion (heavy-duty/box drill/drill.sh), joined to this record by the run ID.\n'
  } > "$out"
}

# =============================================================================
# Pre-flight — every refusal this run can see coming fires here, before
# anything is installed or any credential is spent. Args are validated BEFORE
# the root check (repo doctrine, bootstrap.sh:114 — so the refusals are
# testable without root, and a typo costs a re-type, never a re-ssh).
# =============================================================================
# Both refs EXPLICIT, or nothing runs. Defaulting either to main is exactly
# the #103 hazard this harness exists to refuse: "I drilled the release" must
# not quietly mean "I drilled whatever main was that afternoon".
if [ -z "$REF" ] || [ -z "$BOXREF" ]; then
  echo "drill: both refs must be pinned explicitly — a drill against an unstated ref is not evidence (#103):" >&2
  echo "  --rig-ref <ref>   (or RIG_REF)   the rig candidate, e.g. release/0.4.0   [got: ${REF:-<unset>}]" >&2
  echo "  --box-ref <ref>   (or BOX_REF)   the box that will ship with it          [got: ${BOXREF:-<unset>}]" >&2
  exit 2
fi

case "$ROLE" in
  staging-server|dev-server|control-plane-server|workload-server|runner-server) ;;
  *) echo "drill: --role $ROLE is not a machine role this drill can converge unattended" >&2; exit 2 ;;
esac

if [ -z "$USERS_FILE" ]; then
  echo "drill: --users <path> is required — leg 1 asserts operators converged, and bootstrap requires the file (its --no-users opt-out would leave leg 1 asserting nothing)" >&2
  exit 2
fi
[ -r "$USERS_FILE" ] || { echo "drill: cannot read users file: $USERS_FILE" >&2; exit 2; }

[ "$(id -u)" -eq 0 ] || { echo "drill: must run as root (bootstrap, runner, coolify and db all require it) — ssh in as root on the throwaway machine" >&2; exit 1; }

# The tailnet join needs a key unless this machine already joined (a re-drill
# on the same throwaway). Caught here, not 10 apt-minutes into bootstrap.
if [ -z "${TS_AUTHKEY:-}" ]; then
  if ! { command -v tailscale >/dev/null 2>&1 && tailscale status >/dev/null 2>&1; }; then
    echo "drill: TS_AUTHKEY is unset and this machine has not joined a tailnet — leg 1's bootstrap will refuse. Mint a single-use TAGGED pre-auth key and export TS_AUTHKEY." >&2
    exit 2
  fi
fi

command -v curl >/dev/null 2>&1 || { echo "drill: curl is required (the pinned installs download over it)" >&2; exit 1; }

if [ "$YES" -ne 1 ]; then
  cat <<EOF
This will, ON THIS HOST ($(hostname)):
  · wipe any installed rig and reinstall $REPO@$REF from scratch
  · run 'rig bootstrap $ROLE --users $USERS_FILE' — sshd hardening, hostname
    change, tailnet join, box ($BOXREPO@$BOXREF) + its Incus stack — TWICE
    (the second run is the idempotence assertion)
  · install Coolify${COOLIFY_VERSION:+ $COOLIFY_VERSION}, a GitHub runner${RUNNER_REPO:+ against $RUNNER_REPO} and a Forgejo runner${FJ_INSTANCE:+ against $FJ_INSTANCE}${FJ_RUNNER_REPO:+ ($FJ_RUNNER_REPO)}
Only do this on a THROWAWAY machine you can format.
EOF
  [ -t 0 ] || { echo "drill: no TTY to confirm on — pass --yes if you mean it." >&2; exit 2; }
  printf 'Continue? [y/N] '
  read -r reply
  case "$reply" in y|Y|yes) ;; *) echo "stopped."; exit 1 ;; esac
fi

phase "Pinned candidates"
RIG_SHA="$(ref_sha "$REPO" "$REF")"
BOX_SHA="$(ref_sha "$BOXREPO" "$BOXREF")"
inf "rig: $REPO@$REF (${RIG_SHA:-unresolved})"
inf "box: $BOXREPO@$BOXREF (${BOX_SHA:-unresolved})"
inf "run ID: $RUN_ID — drills sharing this substrate share it (drills/README.md)"

# =============================================================================
phase "Installing rig ($REPO@$REF) from scratch"
# =============================================================================
# The drill proves a tree from SCRATCH every run — a fresh machine, not a
# converged install — so any prior rig goes first (root's install lands at
# \$HOME/.local/share/rig with the /usr/local/bin symlink).
rm -rf "$HOME/.local/share/rig" /usr/local/bin/rig

if ! run_logged /tmp/drill-rig-install.log \
     env RIG_REPO="$REPO" RIG_REF="$REF" \
     bash -c "bash <(curl -fsSL \"https://raw.githubusercontent.com/$REPO/$REF/install.sh\")"; then
  echo "drill: rig's installer failed — tail of /tmp/drill-rig-install.log:" >&2
  tail -5 /tmp/drill-rig-install.log >&2
  exit 1
fi
command -v rig >/dev/null 2>&1 || { echo "drill: installer reported success but no 'rig' on PATH" >&2; exit 1; }

# ASSERT WHAT LANDED — the up-front ref assertion, fatal on mismatch.
RIG_TREE="$(tree_of "$(command -v rig)")"
assert_installed_from rig "$RIG_TREE" "$REPO@$REF" || exit 1
DRILL_VERSION="$(head -n1 "$RIG_TREE/VERSION" 2>/dev/null || echo unknown)"
ok "installed tree confirms: $REPO@$REF (version $DRILL_VERSION)"

# The rig-templates ref this candidate converges (#110), for the record: the
# env override when the drill was pointed somewhere, else the pin read from
# the INSTALLED tree — what actually landed, never this checkout's copy. A
# 40-hex ref IS its own SHA (the pin's normal shape); anything else resolves
# through ref_sha like the two candidates above.
if [ -z "$TPLREF" ]; then
  TPLREF="$(sed -n 's/^RIG_TEMPLATES_PIN=//p' "$RIG_TREE/commands/lib/templates.sh" 2>/dev/null | head -n1)"
  if [ -n "$TPLREF" ] &&
     [ -n "$(find "$RIG_TREE/templates@$TPLREF" -mindepth 2 -maxdepth 2 -type f -name template.env -print -quit 2>/dev/null)" ]; then
    TPL_SOURCE="snapshot"
  fi
fi
if [[ "$TPLREF" =~ ^[0-9a-f]{40}$ ]]; then
  TPL_SHA="${TPLREF:0:7}"
elif [ -n "$TPLREF" ]; then
  TPL_SHA="$(ref_sha "$TPLREPO" "$TPLREF")"
fi
inf "templates: $TPLREPO@${TPLREF:-unresolved} (${TPL_SHA:-unresolved}, $TPL_SOURCE)"
[ -n "$RECORD" ] || RECORD="$ROOT/drills/$DRILL_VERSION.md"

# =============================================================================
phase "Leg 1 — convergence: rig bootstrap $ROLE"
# =============================================================================
# BOX_REPO/BOX_REF ride the environment into bootstrap's host=yes box install,
# so the box that lands is the pinned candidate, not what bootstrap falls back
# to unexported (the BOX_RELEASE pin, since rig#103 landed).
export BOX_REPO="$BOXREPO" BOX_REF="$BOXREF"

t0=$SECONDS
if run_logged /tmp/drill-bootstrap-1.log rig bootstrap "$ROLE" --users "$USERS_FILE"; then
  ok "rig bootstrap $ROLE --users … exited 0  ($((SECONDS - t0))s)"
  BOOTSTRAP_OK=1
else
  no "rig bootstrap $ROLE FAILED — tail: $(tail -3 /tmp/drill-bootstrap-1.log | tr '\n' ' ')"
  BOOTSTRAP_OK=0
fi

MARKER_LINE="$(cat "${RIG_ROLE_MARKER:-/etc/rig/role}" 2>/dev/null || true)"
if [ "$BOOTSTRAP_OK" -eq 1 ]; then
  # The role, asserted on EFFECTIVE state — the marker, the daemon's resolved
  # config, the netmap's granted tags — never on what was requested (the
  # sshd-first-wins lesson, lib/sshd.sh:63-70).
  case "$MARKER_LINE" in
    "role=$ROLE "*) ok "role marker: $MARKER_LINE" ;;
    *) no "role marker is '$MARKER_LINE' — expected role=$ROLE …" ;;
  esac
  sshd -T 2>/dev/null | grep -qx 'passwordauthentication no' \
    && ok "sshd -T resolves passwordauthentication no (the hardening took)" \
    || no "sshd still resolves password auth — the 00-rig.conf drop-in is not winning"
  ts_tags="$(tailscale status --json 2>/dev/null | tr -d '\n ' | grep -o '"Tags":\[[^]]*\]' | head -n1)"
  if [ -n "$ts_tags" ] && [ "$ts_tags" != '"Tags":[]' ]; then
    ok "tailnet joined, tagged: $ts_tags"
  else
    no "tailnet join did not leave a tagged node (got: ${ts_tags:-nothing}) — bootstrap's verify should have refused this"
  fi
  grep -q 'Unattended-Upgrade "1"' /etc/apt/apt.conf.d/20auto-upgrades 2>/dev/null \
    && ok "unattended-upgrades enabled" || no "20auto-upgrades missing or wrong"
  grep -q "converged_by=$DRILL_VERSION" "${RIG_MANIFEST:-/etc/rig/manifest}" 2>/dev/null \
    && ok "manifest: converged_by=$DRILL_VERSION" || no "manifest does not name $DRILL_VERSION as the converging rig"
  users_bad=""
  while read -r u state; do
    [ "$state" = active ] || continue
    id -u "$u" >/dev/null 2>&1 || { users_bad="$users_bad $u(no-account)"; continue; }
    uhome="$(getent passwd "$u" | cut -d: -f6)"
    [ -s "$uhome/.ssh/authorized_keys" ] || users_bad="$users_bad $u(no-keys)"
  done < <(cat "${DRILL_LEDGER:-/etc/rig/users}" 2>/dev/null)
  # NOT 'grep -c … || echo 0': grep -c already prints 0 on no match (and then
  # exits 1), so the fallback would emit a second line into the substitution.
  n_users="$(grep -c ' active$' "${DRILL_LEDGER:-/etc/rig/users}" 2>/dev/null)" || true
  n_users="${n_users:-0}"
  [ -z "$users_bad" ] && [ "$n_users" -gt 0 ] \
    && ok "operators converged: $n_users active, accounts and keys present" \
    || no "operators NOT converged:${users_bad:- ledger empty}"
  leg "convergence — bootstrap $ROLE reaches its role" \
    "$([ "$fail" -eq 0 ] && echo "PASS ($((SECONDS - t0))s)" || echo "FAIL — see Failed below")"

  # --- idempotence: the claim this drill exists to make ----------------------
  # Capture, re-run, capture, diff. Mechanically — never "watched it not
  # obviously break". An empty diff IS the definition of converged.
  phase "Leg 1 — idempotence: the re-run must change nothing"
  pre="$(mktemp)"; post="$(mktemp)"
  capture_state "$pre"
  t0=$SECONDS
  if run_logged /tmp/drill-bootstrap-2.log rig bootstrap "$ROLE" --users "$USERS_FILE"; then
    ok "second bootstrap exited 0  ($((SECONDS - t0))s)"
  else
    no "second bootstrap FAILED — tail: $(tail -3 /tmp/drill-bootstrap-2.log | tr '\n' ' ')"
  fi
  capture_state "$post"
  if statediff="$(diff -u "$pre" "$post")"; then
    ok "re-converge is a no-op: the state diff is empty"
    leg "re-converge (idempotence)" "clean, no changes"
  else
    dlines="$(printf '%s\n' "$statediff" | grep -c '^[+-][^+-]')"
    no "re-converge CHANGED the box — $dlines state line(s) differ:"
    printf '%s\n' "$statediff" | sed 's/^/          /'
    leg "re-converge (idempotence)" "DIRTY — $dlines state line(s) changed on the re-run"
  fi
  rm -f "$pre" "$post"
else
  leg "convergence — bootstrap $ROLE reaches its role" "FAIL — bootstrap exited non-zero"
  skip "idempotence not asserted — the first converge already failed, a re-run diff would measure noise"
  leg "re-converge (idempotence)" "SKIPPED — first converge failed"
fi

# =============================================================================
phase "--host yes — the box that will ship"
# =============================================================================
# The assertions #105 settles this leg at: the installer ran, INSTALLED_FROM
# matches the requested BOX_REF, setup-host exited clean, the stack it claims
# stands. Then it STOPS. Not one isolation probe: two records that both claim
# the trust boundary will eventually disagree with no tiebreaker, and a
# partial isolation check reads — months later, in a record — as though the
# boundary was drilled (box#153's shape through a different door). Resist
# adding "just one" probe here; that is box's drill's whole job.
case "$MARKER_LINE" in
  *"host=yes"*)
    if command -v box >/dev/null 2>&1; then
      ok "box CLI on PATH"
      BOX_TREE="$(tree_of "$(command -v box)")"
      # Fatal, like rig's own: a wrong box under --host yes poisons the pair.
      assert_installed_from box "$BOX_TREE" "$BOXREPO@$BOXREF" || exit 1
      ok "installed box confirms: $BOXREPO@$BOXREF"
      if box doctor >/dev/null 2>&1; then
        ok "box doctor passes — setup-host converged; the host stack stands (box's own effective-state verdict)"
        leg "--host yes: pinned box installed, host stack up" "PASS — $BOXREPO@$BOXREF, box doctor clean"
      else
        no "box is installed but 'box doctor' does not pass — the host stack is unproven (run 'box doctor' for box's verdict)"
        leg "--host yes: pinned box installed, host stack up" "FAIL — box doctor does not pass"
      fi
    else
      no "no 'box' on PATH after a host=yes bootstrap — the box install did not take (bootstrap warns rather than dies there; the drill does not)"
      leg "--host yes: pinned box installed, host stack up" "FAIL — box CLI never landed"
    fi
    inf "isolation NOT asserted here — deliberately. The VM trust boundary is box's"
    inf "assertion, made by box's own drill (~85 probes); this leg stops at 'the pinned"
    inf "box installed and its host stack stands'. The records join on the run ID."
    ;;
  *)
    skip "--host yes assertions: role $ROLE left host=no (marker: ${MARKER_LINE:-absent})"
    leg "--host yes: pinned box installed, host stack up" "SKIPPED — this role does not host VMs"
    ;;
esac

# =============================================================================
phase "Leg 4 — coolify install (pinned, AUTOUPDATE=false)"
# =============================================================================
# Runs BEFORE leg 2 on purpose: Coolify's installer is what puts Docker on the
# box, and the db leg needs a daemon — ordering them the other way around
# would manufacture a skip this same run could have avoided.
if [ -z "$COOLIFY_VERSION" ]; then
  skip "coolify install: no --coolify-version pin given — the leg did not run (rig's own install refuses to default a version, and so does its drill)"
  leg "coolify install" "SKIPPED — no version pin provided"
else
  t0=$SECONDS
  if run_logged /tmp/drill-coolify.log rig coolify install --version "$COOLIFY_VERSION"; then
    ok "rig coolify install --version $COOLIFY_VERSION exited 0  ($((SECONDS - t0))s)"
    grep -qx 'AUTOUPDATE=false' /data/coolify/source/.env 2>/dev/null \
      && ok "AUTOUPDATE=false landed in /data/coolify/source/.env — the platform will not move under its operators" \
      || no "AUTOUPDATE=false is NOT in coolify's .env — the pin is not holding"
    cstate="$(docker inspect -f '{{.State.Status}}' coolify 2>/dev/null || echo absent)"
    [ "$cstate" = running ] && ok "the coolify container is running" \
                            || no "coolify container state: $cstate (expected running)"
    leg "coolify install ($COOLIFY_VERSION)" \
      "$([ "$cstate" = running ] && echo "PASS ($(((SECONDS - t0) / 60)) min)" || echo "FAIL — container $cstate")"
  else
    no "coolify install FAILED — tail: $(tail -3 /tmp/drill-coolify.log | tr '\n' ' ')"
    leg "coolify install ($COOLIFY_VERSION)" "FAIL — installer exited non-zero"
  fi
fi

# =============================================================================
phase "Leg 2 — db dump/restore round-trip (test/db-integration.sh)"
# =============================================================================
# Driven from the INSTALLED tree — the drill exercises what shipped, not the
# checkout this script happens to sit in. The leg's skip contract is the
# script's own (loud, reasoned, exit 0) and classify_leg keeps it a SKIP:
# counted, rendered distinctly, named in the record — never a pass.
db_out="$(mktemp)"
bash "$RIG_TREE/test/db-integration.sh" >"$db_out" 2>&1
db_rc=$?
case "$(classify_leg "$db_rc" "$db_out")" in
  pass)
    db_numbers="$(tail -1 "$db_out")"
    ok "db round-trip: $db_numbers"
    leg "test/db-integration.sh" "PASS — $db_numbers"
    ;;
  skip)
    db_reason="$(grep -m1 '^skip:' "$db_out")"
    skip "db round-trip did not run — $db_reason"
    leg "test/db-integration.sh" "SKIPPED — ${db_reason#skip: }"
    ;;
  fail)
    no "db round-trip FAILED (exit $db_rc) — tail: $(tail -3 "$db_out" | tr '\n' ' ')"
    leg "test/db-integration.sh" "FAIL — exit $db_rc"
    ;;
esac
rm -f "$db_out"

# =============================================================================
phase "Leg 3 — runner lifecycle against a fork"
# =============================================================================
# Register, take a job, deregister. The fork must carry a workflow_dispatch
# workflow (default drill.yml) whose job runs-on the 'drill' label — see
# drill/README.md. Tokens: RUNNER_TOKEN / RUNNER_REMOVE_TOKEN env, or minted
# via an authenticated gh. Without a fork or a token source the leg SKIPS,
# loudly, and the record says it did not run.
if [ -z "$RUNNER_REPO" ]; then
  skip "runner lifecycle: no --runner-repo fork given — the leg did not run"
  leg "runner lifecycle" "SKIPPED — no fork provided"
else
  GH_OK=0
  command -v gh >/dev/null 2>&1 && gh auth status >/dev/null 2>&1 && GH_OK=1
  reg_token="${RUNNER_TOKEN:-}"
  if [ -z "$reg_token" ] && [ "$GH_OK" -eq 1 ]; then
    reg_token="$(gh api -X POST "repos/$RUNNER_REPO/actions/runners/registration-token" --jq .token 2>/dev/null)"
  fi
  if [ -z "$reg_token" ]; then
    skip "runner lifecycle: no RUNNER_TOKEN and no authenticated gh to mint one — the leg did not run"
    leg "runner lifecycle ($RUNNER_REPO)" "SKIPPED — no registration token source"
  else
    RUNNER_NAME="drill-$(hostname)-$$"
    if RUNNER_TOKEN="$reg_token" run_logged /tmp/drill-runner-install.log \
         rig runner install --repo "$RUNNER_REPO" --name "$RUNNER_NAME" --labels drill; then
      ok "rig runner install --repo $RUNNER_REPO exited 0 (registered as $RUNNER_NAME)"
    else
      no "runner install FAILED — tail: $(tail -3 /tmp/drill-runner-install.log | tr '\n' ' ')"
    fi
    rig runner status 2>/dev/null | grep -q "$RUNNER_REPO" \
      && ok "runner status names the fork: $RUNNER_REPO" \
      || no "runner status does not name $RUNNER_REPO"

    took_job=none
    if [ "$GH_OK" -eq 1 ]; then
      # Dispatch, then poll the newest run of that workflow to completion.
      # The newest run's ID is read BEFORE dispatching, so an old completed
      # run can never be mistaken for the one just dispatched (the poll's
      # verdict must be about OUR run, and workflow_dispatch takes a few
      # seconds to materialize a run at all). ~5 min bound: a queued-forever
      # run means the runner never picked the job up, which is exactly what
      # this check exists to catch.
      pre_id="$(gh run list -R "$RUNNER_REPO" --workflow "$RUNNER_WORKFLOW" --limit 1 --json databaseId --jq '.[0].databaseId' 2>/dev/null)"
      if gh workflow run "$RUNNER_WORKFLOW" -R "$RUNNER_REPO" >/dev/null 2>&1; then
        inf "dispatched $RUNNER_WORKFLOW on $RUNNER_REPO — waiting for the runner to take it (≤5 min)…"
        took_job=timeout
        for _i in $(seq 1 30); do
          sleep 10
          run_line="$(gh run list -R "$RUNNER_REPO" --workflow "$RUNNER_WORKFLOW" --limit 1 \
            --json databaseId,status,conclusion --jq '.[0] | "\(.databaseId) \(.status) \(.conclusion)"' 2>/dev/null)"
          read -r rid rstatus rconc <<< "$run_line"
          [ -n "${rid:-}" ] || continue
          [ "$rid" != "${pre_id:-}" ] || continue
          if [ "${rstatus:-}" = completed ]; then
            case "${rconc:-}" in
              success) took_job=success ;;
              *) took_job=failed ;;
            esac
            break
          fi
        done
      else
        took_job=nodispatch
      fi
      case "$took_job" in
        success) ok "the runner took a job and it succeeded ($RUNNER_WORKFLOW)" ;;
        failed)  no "the dispatched job completed UNSUCCESSFULLY — the runner ran it, the workflow failed; read the run on $RUNNER_REPO" ;;
        timeout) no "the dispatched job never completed within 5 min — the runner did not take it (is the workflow's runs-on label 'drill'?)" ;;
        nodispatch) no "could not dispatch $RUNNER_WORKFLOW on $RUNNER_REPO — does the fork carry it, with workflow_dispatch? (see drill/README.md)" ;;
      esac
    else
      skip "took a job: not attempted — no authenticated gh to dispatch $RUNNER_WORKFLOW with"
    fi

    rem_token="${RUNNER_REMOVE_TOKEN:-}"
    if [ -z "$rem_token" ] && [ "$GH_OK" -eq 1 ]; then
      rem_token="$(gh api -X POST "repos/$RUNNER_REPO/actions/runners/remove-token" --jq .token 2>/dev/null)"
    fi
    if [ -n "$rem_token" ]; then
      RUNNER_REMOVE_TOKEN="$rem_token" rig runner remove >/dev/null 2>&1 \
        && ok "rig runner remove deregistered cleanly" \
        || no "runner remove FAILED"
    else
      rig runner remove --local >/dev/null 2>&1 \
        && note "deregistered --local only (no removal token source) — delete the stale runner from $RUNNER_REPO's settings by hand" \
        || no "runner remove --local FAILED"
    fi
    rig runner status >/dev/null 2>&1 \
      && no "runner status still answers after remove — the deregistration did not take" \
      || ok "runner status confirms: nothing registered"

    leg "runner lifecycle ($RUNNER_REPO)" \
      "$(case "$took_job" in
           success) echo "PASS — registered, took a job, deregistered clean" ;;
           none)    echo "PARTIAL — registered and deregistered; took a job: not attempted (no gh)" ;;
           *)       echo "FAIL — see Failed below" ;;
         esac)"
  fi
fi

# =============================================================================
phase "Leg 3 — forgejo runner lifecycle against an instance"
# =============================================================================
# The same leg as above, for the other forge. Both forges ship a runner family
# (#109 added `rig forgejo-runner` beside `rig runner`), so a release that
# evidences only GitHub evidences half of what it ships (#129).
#
# Three things differ from the GitHub leg, all measured against
# forgejo.heavyduty.builders (8.0.3+gitea-1.22.0) on 2026-07-30, not read:
#
#   * Scope is the TOKEN's, never a flag — `rig forgejo-runner install` refuses
#     --repo on purpose (commands/forgejo-runner-install.sh:160). The repo here
#     is only where the registration token is minted from, and where the
#     dispatched workflow lives.
#   * The mint path is /repos/<o>/<r>/actions/runners/registration-token. The
#     instance's own swagger documents /repos/<o>/<r>/runners/registration-token
#     — WITHOUT /actions/ — and that path 404s. Do not "fix" this to match the
#     published API reference.
#   * There is no deregistration endpoint, so there is no removal token and no
#     remote deregistration: `rig forgejo-runner remove` is local-only by
#     design (commands/forgejo-runner-remove.sh:7-11) and the runner row
#     survives in the UI until a human deletes it. The record says so rather
#     than implying a clean remote teardown the way the GitHub leg can.
#
# Tokens: FORGEJO_RUNNER_TOKEN (a registration token) is used directly; else
# FORGEJO_API_TOKEN mints one from FJ_RUNNER_REPO. Without an instance, a repo,
# or a token source the leg SKIPS loudly and the record says it did not run.
if [ -z "$FJ_INSTANCE" ] || [ -z "$FJ_RUNNER_REPO" ]; then
  skip "forgejo runner lifecycle: no --forgejo-instance/--forgejo-runner-repo given — the leg did not run"
  leg "forgejo runner lifecycle" "SKIPPED — no instance/repo provided"
else
  fj_reg="${FORGEJO_RUNNER_TOKEN:-}"
  if [ -z "$fj_reg" ] && [ -n "${FORGEJO_API_TOKEN:-}" ]; then
    fj_reg="$(curl -fsSL -H "Authorization: token ${FORGEJO_API_TOKEN}" \
      "${FJ_INSTANCE%/}/api/v1/repos/${FJ_RUNNER_REPO}/actions/runners/registration-token" 2>/dev/null \
      | grep -o '"token"[[:space:]]*:[[:space:]]*"[^"]*"' | head -n1 \
      | sed 's/.*:[[:space:]]*"//; s/"$//')"
  fi
  fj_tok_verdict="$(forgejo_token_verdict "$fj_reg" "${FORGEJO_API_TOKEN:-}")"
  if [ "$fj_tok_verdict" = mint-failed ]; then
    # Configured, and it did not work. Never a skip: see forgejo_token_verdict.
    no "registration-token mint FAILED against ${FJ_INSTANCE} — is it reachable, and does FORGEJO_API_TOKEN own ${FJ_RUNNER_REPO}? (the token is never printed)"
    leg "forgejo runner lifecycle ($FJ_RUNNER_REPO)" "FAIL — registration-token mint failed"
  elif [ "$fj_tok_verdict" = no-source ]; then
    skip "forgejo runner lifecycle: no FORGEJO_RUNNER_TOKEN and no FORGEJO_API_TOKEN to mint one — the leg did not run"
    leg "forgejo runner lifecycle ($FJ_RUNNER_REPO)" "SKIPPED — no registration token source"
  else
    FJ_NAME="drill-$(hostname)-$$"
    fj_install_ok=0 fj_status_ok=0 fj_remove_ok=0 fj_absent_ok=0
    # The label MUST carry a docker:// image: forgejo-runner runs jobs in
    # containers, and a bare label leaves runs-on matched but unrunnable.
    if FORGEJO_RUNNER_TOKEN="$fj_reg" run_logged /tmp/drill-forgejo-runner-install.log \
         rig forgejo-runner install --instance "$FJ_INSTANCE" --name "$FJ_NAME" \
         --labels 'drill:docker://node:22-bookworm'; then
      ok "rig forgejo-runner install --instance $FJ_INSTANCE exited 0 (registered as $FJ_NAME)"
      fj_install_ok=1
    else
      no "forgejo-runner install FAILED — tail: $(tail -3 /tmp/drill-forgejo-runner-install.log | tr '\n' ' ')"
    fi
    if rig forgejo-runner status 2>/dev/null | grep -qF "${FJ_INSTANCE%/}"; then
      ok "forgejo-runner status names the instance: $FJ_INSTANCE"; fj_status_ok=1
    else
      no "forgejo-runner status does not name ${FJ_INSTANCE}"
    fi

    fj_took=none
    # Do NOT dispatch once install or status has failed. The job would be taken
    # by whatever else answers runs-on: drill — a stale runner this leg's own
    # hand-delete caveat leaves behind — and its success would be evidence about
    # someone else's runner (codex/grok/kimi, !130).
    if [ "$fj_install_ok" != 1 ] || [ "$fj_status_ok" != 1 ]; then
      skip "took a job: not attempted — install or status failed, and a foreign runner answering 'drill' could only manufacture a false pass"
    elif [ -n "${FORGEJO_API_TOKEN:-}" ]; then
      fj_api="${FJ_INSTANCE%/}/api/v1/repos/${FJ_RUNNER_REPO}"
      # Read the newest ASSIGNED task id BEFORE dispatching, same guard as the
      # GitHub leg: an already-completed run must never be read as ours.
      fj_pre_body="$(curl -fsSL -H "Authorization: token ${FORGEJO_API_TOKEN}" \
        "$fj_api/actions/tasks" 2>/dev/null || echo '{}')"
      printf '%s' "$fj_pre_body" > /tmp/drill-forgejo-pre.json
      fj_pre="$(forgejo_max_task_id /tmp/drill-forgejo-pre.json)"
      if curl -fsSL -o /dev/null -X POST -H "Authorization: token ${FORGEJO_API_TOKEN}" \
           -H "Content-Type: application/json" -d "{\"ref\":\"${FJ_REF}\"}" \
           "$fj_api/actions/workflows/${RUNNER_WORKFLOW}/dispatches" 2>/dev/null; then
        inf "dispatched $RUNNER_WORKFLOW on $FJ_RUNNER_REPO — waiting for the runner to take it (≤5 min)…"
        fj_took=timeout
        for _i in $(seq 1 30); do
          sleep 10
          curl -fsSL -H "Authorization: token ${FORGEJO_API_TOKEN}" \
            "$fj_api/actions/tasks" -o /tmp/drill-forgejo-tasks.json 2>/dev/null || continue
          case "$(forgejo_run_verdict "${fj_pre:-}" /tmp/drill-forgejo-tasks.json)" in
            success) fj_took=success; break ;;
            failed)  fj_took=failed;  break ;;
            *) : ;;   # pending — queued, or assigned and still running
          esac
        done
      else
        fj_took=nodispatch
      fi
      case "$fj_took" in
        success) ok "the forgejo runner took a job and it succeeded ($RUNNER_WORKFLOW)" ;;
        failed)  no "the dispatched job completed UNSUCCESSFULLY — the runner ran it, the workflow failed; read the run on $FJ_RUNNER_REPO" ;;
        # A queued task is INVISIBLE in this API until a runner claims it, so a
        # timeout means "nothing ever took it". Two causes, and the second one
        # is not rig's: the runs-on label may not match, or the daemon's poller
        # can go quiet — a restarted daemon claims a minutes-old backlog in
        # about a second. Check 'systemctl restart forgejo-runner' before
        # reading this as a rig defect.
        timeout) no "the dispatched job was never taken within 5 min — check the workflow's runs-on is 'drill', then restart forgejo-runner and re-read (a quiet poller looks exactly like this)" ;;
        nodispatch) no "could not dispatch $RUNNER_WORKFLOW on $FJ_RUNNER_REPO — does it carry that workflow, with workflow_dispatch, on its default branch?" ;;
      esac
    else
      skip "took a job: not attempted — no FORGEJO_API_TOKEN to dispatch $RUNNER_WORKFLOW with"
    fi

    # No removal token exists on this forge — remove is local by design.
    if rig forgejo-runner remove >/dev/null 2>&1; then
      note "forgejo-runner removed locally — Forgejo has no deregistration endpoint, so DELETE the stale '$FJ_NAME' row under $FJ_RUNNER_REPO > Settings > Actions > Runners by hand"
      fj_remove_ok=1
    else
      no "forgejo-runner remove FAILED"
    fi
    if rig forgejo-runner status >/dev/null 2>&1; then
      no "forgejo-runner status still answers after remove — the removal did not take"
    else
      ok "forgejo-runner status confirms: nothing registered"; fj_absent_ok=1
    fi

    leg "forgejo runner lifecycle ($FJ_RUNNER_REPO)" \
      "$(forgejo_leg_row "$fj_install_ok" "$fj_status_ok" "$fj_took" \
                         "$fj_remove_ok" "$fj_absent_ok")"
  fi
fi

# =============================================================================
phase "Summary"
# =============================================================================
printf '  %s passed, %s failed, %s skipped\n' "$pass" "$fail" "$skipped"
if [ "${#findings[@]}" -gt 0 ]; then
  echo
  printf '  %s\n' "${findings[@]}"
fi

mkdir -p "$(dirname "$RECORD")"
emit_record "$RECORD"
echo
inf "record written: $RECORD"
inf "commit it on the release branch as drills/$DRILL_VERSION.md — the"
inf "drill-recorded gate reads that file and nothing else (drills/README.md)."
[ "$fail" -eq 0 ]