codex-bot-andresmgsl/rig
1
0
Fork 0
forked from heavy-duty/rig
Code Pull requests Activity
rig/test/drill.sh
cluade-reviewer-andresmgsl 1179d3142f feat(drill): a forgejo-runner lifecycle leg beside the GitHub one 
#109 shipped `rig forgejo-runner` and the drill never touched it, so the
release gate could pass with the whole family broken. Its only coverage was
15 argument-handling checks in test/cli.sh.

The Forgejo half cannot be a copy of the GitHub one. Measured against
forgejo.heavyduty.builders (8.0.3+gitea-1.22.0), not read from the docs:
GET actions/tasks carries NO `conclusion` field — `status` holds the terminal
outcome — `id` is a global task id while the URL ends in run_number, and the
payload lists ASSIGNED tasks only, so a queued run is invisible (measured:
200s of total_count 0 while the web UI showed the run). That makes "no new
id" the only signal the runner never took the job, which is the verdict this
leg exists to produce.

forgejo_run_verdict() is the testable core of that, grep-and-sed rather than
jq for the same reason json_field() is. Tests were written first and cover
the cases that must NOT pass: a stale run with the pre-dispatch id, an
assigned-but-still-running task, and a cancelled run.

Refs #129

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-30 23:27:36 +00:00

279 lines
16 KiB
Bash
Raw Blame History

#!/usr/bin/env bash
# test/drill.sh — the drill harness's HONESTY, proven without hardware.
#
# drill/drill.sh is the instrument (#105), so what this suite tests is the
# instrument itself: the refusals, the classifications, the capture-and-diff
# that decides idempotence, and the record emitter — the parts whose lies
# would be believed, months later, by a reader of drills/<version>.md. The
# four-leg live run on a real Debian machine is #107's exercise, not this
# file's: nothing here needs root, Docker, a tailnet or the network.
#
# Extraction pattern is test/release.sh's: the functions under test are
# awk-extracted from drill/drill.sh and driven against fixtures, so the tests
# exercise the shipped bytes, and the extraction check itself guards the awk
# against a drifted function boundary.
# Deliberately no `set -e` — the harness asserts on failing commands.
set -u
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT" || exit 1
PASS=0 FAIL=0
# check <desc> <want_exit> <want_substr> <cmd...>
check() {
local desc="$1" want="$2" substr="$3"; shift 3
local out rc
out="$("$@" 2>&1)"; rc=$?
if [ "$rc" -ne "$want" ]; then
echo "FAIL: $desc — exit $rc, wanted $want"
printf '%s\n' "$out" | sed 's/^/ /'
FAIL=$((FAIL + 1)); return
fi
if [ -n "$substr" ] && ! printf '%s' "$out" | grep -qF -e "$substr"; then
echo "FAIL: $desc — output missing '$substr'"
printf '%s\n' "$out" | sed 's/^/ /'
FAIL=$((FAIL + 1)); return
fi
echo "ok: $desc"; PASS=$((PASS + 1))
}
# refute <desc> <substr> <file> — the file must NOT contain the substring.
refute() {
if grep -qF -e "$2" "$3"; then
echo "FAIL: $1 — found forbidden '$2'"
FAIL=$((FAIL + 1)); return
fi
echo "ok: $1"; PASS=$((PASS + 1))
}
WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT
# --- the functions under test, extracted -------------------------------------
FNS="$WORK/drill-fns.sh"
for fn in tree_of assert_installed_from classify_leg capture_state emit_record forgejo_run_verdict; do
awk "/^${fn}\(\) \{/,/^\}/" "$ROOT/drill/drill.sh" >> "$FNS"
done
for fn in tree_of assert_installed_from classify_leg capture_state emit_record forgejo_run_verdict; do
check "extraction guards the awk: ${fn}() landed" 0 "${fn}() {" grep -F "${fn}() {" "$FNS"
done
# shellcheck source=/dev/null
. "$FNS"
# =============================================================================
# tree_of — the versioned tree behind a CLI's symlink chain
# =============================================================================
IR="$WORK/install"; mkdir -p "$IR/versions/1.2.3/bin"
: > "$IR/versions/1.2.3/bin/rig"
ln -s "versions/1.2.3" "$IR/current"
mkdir -p "$WORK/bin"
ln -s "$IR/current/bin/rig" "$WORK/bin/rig"
check "tree_of resolves a current-symlink chain to versions/<v>" 0 "$IR/versions/1.2.3" \
tree_of "$WORK/bin/rig"
ln -s "$IR/gone/bin/rig" "$WORK/bin/dangling"
check "tree_of refuses a dangling chain — a tree that is not there is not a tree" 1 "" \
tree_of "$WORK/bin/dangling"
# =============================================================================
# assert_installed_from — the up-front ref refusal, naming both refs
# =============================================================================
TREE="$WORK/tree-main"; mkdir -p "$TREE"
printf 'heavy-duty/rig@main\n' > "$TREE/INSTALLED_FROM"
check "matching INSTALLED_FROM passes silently" 0 "" \
assert_installed_from rig "$TREE" "heavy-duty/rig@main"
check "a mismatch refuses (the #103 hazard: asked release, got main)" 1 "FATAL" \
assert_installed_from rig "$TREE" "heavy-duty/rig@release/9.9.9"
check "…the refusal names the ref that was ASKED for" 1 "heavy-duty/rig@release/9.9.9" \
assert_installed_from rig "$TREE" "heavy-duty/rig@release/9.9.9"
check "…and the ref that actually LANDED" 1 "heavy-duty/rig@main" \
assert_installed_from rig "$TREE" "heavy-duty/rig@release/9.9.9"
check "an unreadable INSTALLED_FROM refuses too — absence is not a match" 1 "<unreadable>" \
assert_installed_from rig "$WORK/no-such-tree" "heavy-duty/rig@main"
# =============================================================================
# classify_leg — a loud skip is a SKIP, never a pass (box#153's defect class)
# =============================================================================
printf 'skip: docker not installed — nothing to exercise\n' > "$WORK/out-skip"
printf 'ok: seeded\nok: restored\n---\n14 passed, 0 failed\n' > "$WORK/out-pass"
printf 'FAIL: restore blew up\n' > "$WORK/out-fail"
check "exit 0 + 'skip:' line classifies as skip" 0 "skip" classify_leg 0 "$WORK/out-skip"
check "exit 0, no skip line, classifies as pass" 0 "pass" classify_leg 0 "$WORK/out-pass"
check "non-zero exit classifies as fail" 0 "fail" classify_leg 1 "$WORK/out-fail"
check "a skip line cannot rescue a non-zero exit (fail wins)" 0 "fail" \
classify_leg 1 "$WORK/out-skip"
# =============================================================================
# capture_state + diff — the idempotence verdict's machinery. The claim in
# #105's acceptance criteria: the assertion is a REAL diff of captured state,
# and it FAILS when convergence is broken — demonstrated here, mechanically,
# on every CI run, by breaking the state between two captures.
# =============================================================================
FIX="$WORK/fix"; mkdir -p "$FIX/sudoers.d"
printf 'role=staging-server root-door=open host=yes join=authkey\n' > "$FIX/role"
printf 'schema=1\nbootstrapped_by=9.9.9\nbootstrapped_at=T\nconverged_by=9.9.9\nconverged_at=T\n' > "$FIX/manifest"
printf 'dan active\nghost revoked\n' > "$FIX/ledger"
printf 'APT::Periodic::Update-Package-Lists "1";\n' > "$FIX/autoup"
printf '127.0.0.1 localhost\n127.0.1.1\tstaging-server\n' > "$FIX/hosts"
printf 'nosuchdrilluser ALL=(ALL) NOPASSWD:ALL\n' > "$FIX/sudoers.d/00-rig-nosuch"
# A stubbed sshd, so the effective-config section is exercised rather than
# skipped on a box with no daemon (repo precedent: test/release.sh's curl).
STUB="$WORK/stub"; mkdir -p "$STUB"
# The single-quoted $SSHD_FIXTURE is the STUB's expansion, not this shell's.
# shellcheck disable=SC2016
printf '#!/usr/bin/env bash\ncat "$SSHD_FIXTURE"\n' > "$STUB/sshd"; chmod +x "$STUB/sshd"
printf 'passwordauthentication no\npermitrootlogin prohibit-password\n' > "$FIX/sshd-T"
cap() { # cap <outfile> — capture_state against the fixture set
RIG_ROLE_MARKER="$FIX/role" RIG_MANIFEST="$FIX/manifest" \
DRILL_LEDGER="$FIX/ledger" DRILL_AUTOUPGRADES="$FIX/autoup" \
DRILL_ETC_HOSTS="$FIX/hosts" DRILL_SUDOERS_DIR="$FIX/sudoers.d" \
SSHD_FIXTURE="$FIX/sshd-T" PATH="$STUB:$PATH" \
bash -c '. "$1"; capture_state "$2"' _ "$FNS" "$2" 2>/dev/null
:
}
# cap runs capture_state in a child bash so the PATH stub cannot leak into
# this harness; $2 arrives as the capture's outfile.
cap out "$WORK/cap1"
cap out "$WORK/cap2"
check "two captures over untouched state diff EMPTY (the converged verdict)" 0 "" \
diff -u "$WORK/cap1" "$WORK/cap2"
check "the capture reads the fixtures, not the machine (marker line present)" 0 "role=staging-server" \
grep -o 'role=staging-server[^"]*' "$WORK/cap1"
check "…the sshd section captured the effective config" 0 "passwordauthentication no" \
cat "$WORK/cap1"
check "…a ledger user with no account reads as one, deterministically" 0 "(no account)" \
cat "$WORK/cap1"
# Break convergence: the re-run "changed" the role marker and root's door.
printf 'role=staging-server root-door=closed host=yes join=authkey\n' > "$FIX/role"
printf 'passwordauthentication yes\npermitrootlogin prohibit-password\n' > "$FIX/sshd-T"
cap out "$WORK/cap3"
check "a broken convergence makes the diff NON-empty — the assertion can fail" 1 "root-door=closed" \
diff -u "$WORK/cap1" "$WORK/cap3"
check "…and the diff names the drifted sshd keyword, not just 'differs'" 1 "passwordauthentication yes" \
diff -u "$WORK/cap1" "$WORK/cap3"
# =============================================================================
# emit_record — the record is drills/README.md's shape, and it cannot lie:
# a failed run still emits, a skipped leg is named, no clean-sweep reading.
# =============================================================================
emit() { # emit <outfile> — emit_record with the harness globals staged
DRILL_VERSION="9.9.9" RUN_ID="drill-2026-01-01-a" \
REF="release/9.9.9" BOXREF="release/0.4.0" RIG_SHA="5d6e7f8" BOX_SHA="1a2b3c4" \
TPLREPO="heavy-duty/rig-templates" TPLREF="9f8e7d6c5b4a39281706f5e4d3c2b1a098765432" TPL_SHA="9f8e7d6" TPL_SOURCE="snapshot" \
bash -c '
. "$1"
pass=12 fail=1 skipped=1
findings=("FAIL: coolify container state: absent" "SKIP: runner lifecycle: no --runner-repo fork given — the leg did not run" "NOTE: something worth a line")
LEG_NAMES=("convergence — bootstrap staging-server reaches its role" "re-converge (idempotence)" "coolify install (4.1.2)" "runner lifecycle")
LEG_RESULTS=("PASS (312s)" "clean, no changes" "FAIL — container absent" "SKIPPED — no fork provided")
emit_record "$2"
' _ "$FNS" "$2"
}
emit out "$WORK/record.md"
check "record: the version-and-date heading" 0 "# Release drill — 9.9.9 — " head -1 "$WORK/record.md"
check "record: the run ID that joins the family's records" 0 "Run ID: drill-2026-01-01-a" cat "$WORK/record.md"
check "record: both pinned refs with their SHAs" 0 "rig@5d6e7f8 (RIG_REF=release/9.9.9)" cat "$WORK/record.md"
check "record: …box's too" 0 "box@1a2b3c4 (BOX_REF=release/0.4.0)" cat "$WORK/record.md"
check "record: the template registry SHA and actual source ride alongside the pair (#110/#153)" 0 "rig-templates@9f8e7d6 (ref 9f8e7d6c5b4a39281706f5e4d3c2b1a098765432, snapshot)" cat "$WORK/record.md"
check "record: one table row per leg, result verbatim" 0 "| re-converge (idempotence) | clean, no changes |" cat "$WORK/record.md"
check "record: the numbers, skips counted apart from passes" 0 "12 passed, 1 failed, 1 skipped" cat "$WORK/record.md"
check "record: a FAILED run still names what failed (evidence, not success)" 0 "FAIL: coolify container state: absent" cat "$WORK/record.md"
check "record: a skipped leg is stated as NOT run, by name" 0 "SKIP: runner lifecycle" cat "$WORK/record.md"
check "record: the skip section says the record is not evidence for it" 0 "not evidence" cat "$WORK/record.md"
check "record: the isolation boundary is named as box's, in words" 0 "NOT asserted here" cat "$WORK/record.md"
refute "record with a skip cannot read as a clean sweep" "Failed: nothing" "$WORK/record.md"
refute "notes are findings for the log, not failures for the record" "NOTE: something" "$WORK/record.md"
# The all-green shape: says so plainly, and only then.
DRILL_VERSION="9.9.9" RUN_ID="drill-2026-01-01-a" \
REF="release/9.9.9" BOXREF="release/0.4.0" RIG_SHA="5d6e7f8" BOX_SHA="1a2b3c4" \
bash -c '
. "$1"
pass=20 fail=0 skipped=0
findings=()
LEG_NAMES=("convergence" "re-converge (idempotence)")
LEG_RESULTS=("PASS" "clean, no changes")
emit_record "$2"
' _ "$FNS" "$WORK/record-green.md"
check "an all-green record says every leg ran and passed" 0 "Every leg ran and every check passed" \
cat "$WORK/record-green.md"
# =============================================================================
# the shipped script itself
# =============================================================================
# =============================================================================
# forgejo_run_verdict — did OUR dispatched run land, and how (#129)
# =============================================================================
# The Forgejo half of the runner leg cannot reuse the GitHub reader. Measured
# against forgejo.heavyduty.builders (8.0.3+gitea-1.22.0) on 2026-07-30, a
# completed run in GET /repos/{o}/{r}/actions/tasks carries NO `conclusion`
# field at all — `status` holds the terminal outcome directly, where GitHub
# splits status:completed + conclusion:success. And `id` is a global task id
# (25) while the run's own URL ends in run_number (1), so the pre-dispatch
# guard has to compare `id`.
#
# The payload is also an ASSIGNED-task view: it reads total_count 0 for as
# long as a run sits queued (measured: 200s), so "no new id" is the ONLY
# signal that the runner never took the job. That is the verdict this leg
# exists to produce, which is why it gets its own function and its own tests.
FJ="$WORK/fj"; mkdir -p "$FJ"
printf '%s' '{"workflow_runs":[],"total_count":0}' > "$FJ/empty.json"
printf '%s' '{"workflow_runs":[{"id":25,"status":"success","run_number":1,"url":"https://f/o/r/actions/runs/1"}],"total_count":1}' > "$FJ/new-success.json"
printf '%s' '{"workflow_runs":[{"id":25,"status":"failure","run_number":1,"url":"https://f/o/r/actions/runs/1"}],"total_count":1}' > "$FJ/new-failure.json"
printf '%s' '{"workflow_runs":[{"id":25,"status":"cancelled","run_number":1,"url":"https://f/o/r/actions/runs/1"}],"total_count":1}' > "$FJ/new-cancelled.json"
printf '%s' '{"workflow_runs":[{"id":24,"status":"success","run_number":1,"url":"https://f/o/r/actions/runs/1"}],"total_count":1}' > "$FJ/stale-only.json"
check "verdict: an empty task list is PENDING, never a pass" 0 "pending" \
forgejo_run_verdict "" "$FJ/empty.json"
check "verdict: a queued run the runner never took stays PENDING" 0 "pending" \
forgejo_run_verdict "24" "$FJ/stale-only.json"
check "verdict: OUR new run, status success, is SUCCESS" 0 "success" \
forgejo_run_verdict "24" "$FJ/new-success.json"
check "verdict: status carries the outcome — failure is FAILED, not success" 0 "failed" \
forgejo_run_verdict "24" "$FJ/new-failure.json"
check "verdict: a cancelled run is FAILED, not silently passed" 0 "failed" \
forgejo_run_verdict "24" "$FJ/new-cancelled.json"
check "verdict: the first run ever (no pre-id) still resolves" 0 "success" \
forgejo_run_verdict "" "$FJ/new-success.json"
# A task appears in this payload the moment it is ASSIGNED, which can be before
# it finishes — so a non-terminal status must read as pending, not as a failure.
# Calling a still-running job "failed" would make the leg flaky in exactly the
# window the leg is watching.
printf '%s' '{"workflow_runs":[{"id":25,"status":"running","run_number":1}],"total_count":1}' > "$FJ/new-running.json"
check "verdict: an assigned-but-running task is PENDING, not FAILED" 0 "pending" \
forgejo_run_verdict "24" "$FJ/new-running.json"
# The anti-false-positive guard, stated as its own case: an OLD completed run
# with the SAME id as pre_id must never be read as this dispatch's result.
check "verdict: a pre-existing success with the pre-id is NOT our run" 0 "pending" \
forgejo_run_verdict "25" "$FJ/new-success.json"
# Arg refusals fire before the root check (repo doctrine, bootstrap.sh:114),
# which is what makes them provable here without a throwaway machine.
check "drill.sh refuses to run without BOTH refs pinned (#103)" 2 "--box-ref" \
env -u RIG_REF -u BOX_REF bash "$ROOT/drill/drill.sh" --rig-ref release/9.9.9 --yes
check "…and the refusal shows which ref is missing" 2 "<unset>" \
env -u RIG_REF -u BOX_REF bash "$ROOT/drill/drill.sh" --rig-ref release/9.9.9 --yes
check "a tenant role is refused — the drill converges machines, not guests" 2 "not a machine role" \
bash "$ROOT/drill/drill.sh" --rig-ref r --box-ref b --role claude-box --yes
check "no --users is a refusal, naming why the drill will not default it" 2 "--users <path> is required" \
bash "$ROOT/drill/drill.sh" --rig-ref r --box-ref b --yes
check "an unreadable users file dies before anything is spent" 2 "cannot read users file" \
bash "$ROOT/drill/drill.sh" --rig-ref r --box-ref b --users "$WORK/no-such-users" --yes
check "an unknown flag dies loudly, exit 2" 2 "unknown option" \
bash "$ROOT/drill/drill.sh" --frobnicate
check "--help prints the header and exits 0" 0 "THROWAWAY" \
bash "$ROOT/drill/drill.sh" --help
check "--forgejo-instance is a known flag (the leg's opt-in)" 2 "--users <path> is required" \
bash "$ROOT/drill/drill.sh" --rig-ref r --box-ref b --forgejo-instance https://f.example.com --yes
check "--forgejo-runner-repo is a known flag" 2 "--users <path> is required" \
bash "$ROOT/drill/drill.sh" --rig-ref r --box-ref b --forgejo-runner-repo o/r --yes
check "--help names the forgejo runner leg's flags" 0 "--forgejo-instance" \
bash "$ROOT/drill/drill.sh" --help
check "--forgejo-ref is a known flag (Forgejo's dispatch needs a ref)" 2 "--users <path> is required" \
bash "$ROOT/drill/drill.sh" --rig-ref r --box-ref b --forgejo-ref dev --yes
echo "---"
echo "$PASS passed, $FAIL failed"
[ "$FAIL" -eq 0 ]
View git blame Copy permalink