2026-07-24 00:31:53 +00:00
#!/usr/bin/env bash
# test/drill.sh — the drill harness's HONESTY, proven without hardware.
#
# drill/drill.sh is the instrument (#105), so what this suite tests is the
# instrument itself: the refusals, the classifications, the capture-and-diff
# that decides idempotence, and the record emitter — the parts whose lies
# would be believed, months later, by a reader of drills/<version>.md. The
# four-leg live run on a real Debian machine is #107's exercise, not this
# file's: nothing here needs root, Docker, a tailnet or the network.
#
# Extraction pattern is test/release.sh's: the functions under test are
# awk-extracted from drill/drill.sh and driven against fixtures, so the tests
# exercise the shipped bytes, and the extraction check itself guards the awk
# against a drifted function boundary.
# Deliberately no `set -e` — the harness asserts on failing commands.
set -u
ROOT = " $( cd " $( dirname " ${ BASH_SOURCE [0] } " ) /.. " && pwd ) "
cd " $ROOT " || exit 1
PASS = 0 FAIL = 0
# check <desc> <want_exit> <want_substr> <cmd...>
check( ) {
local desc = " $1 " want = " $2 " substr = " $3 " ; shift 3
local out rc
out = " $( " $@ " 2>& 1) " ; rc = $?
if [ " $rc " -ne " $want " ] ; then
echo " FAIL: $desc — exit $rc , wanted $want "
printf '%s\n' " $out " | sed 's/^/ /'
FAIL = $(( FAIL + 1 )) ; return
fi
if [ -n " $substr " ] && ! printf '%s' " $out " | grep -qF -e " $substr " ; then
echo " FAIL: $desc — output missing ' $substr ' "
printf '%s\n' " $out " | sed 's/^/ /'
FAIL = $(( FAIL + 1 )) ; return
fi
echo " ok: $desc " ; PASS = $(( PASS + 1 ))
}
# refute <desc> <substr> <file> — the file must NOT contain the substring.
refute( ) {
if grep -qF -e " $2 " " $3 " ; then
echo " FAIL: $1 — found forbidden ' $2 ' "
FAIL = $(( FAIL + 1 )) ; return
fi
echo " ok: $1 " ; PASS = $(( PASS + 1 ))
}
WORK = " $( mktemp -d) "
trap 'rm -rf "$WORK"' EXIT
# --- the functions under test, extracted -------------------------------------
FNS = " $WORK /drill-fns.sh "
fix(drill): the leg row is the whole lifecycle, not just the job
codex, grok and kimi all landed on the same hole at e42ba31, and they are
right. The record row keyed only on fj_took, so install/status/remove failures
called `no` without gating it. A dispatched job only needs SOMETHING answering
runs-on: drill — and this leg removes locally, telling the operator to delete
the stale runner by hand, so a leftover drill-labeled runner from the previous
drill is the designed-for aftermath, not a contrived case. It answers the
fixture exactly, and the row then read "PASS — registered, took a job,
removed" for a drill whose install never happened.
The exit code was never wrong: every one of those failures also called `no`.
What was wrong is the row, and the row is what outlives the run in
drills/<v>.md. A row claiming a lifecycle that did not happen is exactly what
the gate exists to refuse — kimi's framing, and it is the right one.
forgejo_leg_row folds all five assertions, and the leg no longer dispatches
after install or status has failed: a foreign runner's success could only ever
be evidence about someone else's runner. The regression composes the real
functions over a payload carrying a newer successful run, the way the leg
composes them.
The GitHub leg has the identical shape and is byte-frozen by #129's contract;
left alone deliberately, and worth a sibling issue for triage.
Refs #129
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-31 20:04:16 +00:00
for fn in tree_of assert_installed_from classify_leg capture_state emit_record forgejo_run_verdict forgejo_token_verdict forgejo_max_task_id forgejo_leg_row; do
2026-07-24 00:31:53 +00:00
awk " /^ ${ fn } \(\) \{/,/^\}/ " " $ROOT /drill/drill.sh " >> " $FNS "
done
fix(drill): the leg row is the whole lifecycle, not just the job
codex, grok and kimi all landed on the same hole at e42ba31, and they are
right. The record row keyed only on fj_took, so install/status/remove failures
called `no` without gating it. A dispatched job only needs SOMETHING answering
runs-on: drill — and this leg removes locally, telling the operator to delete
the stale runner by hand, so a leftover drill-labeled runner from the previous
drill is the designed-for aftermath, not a contrived case. It answers the
fixture exactly, and the row then read "PASS — registered, took a job,
removed" for a drill whose install never happened.
The exit code was never wrong: every one of those failures also called `no`.
What was wrong is the row, and the row is what outlives the run in
drills/<v>.md. A row claiming a lifecycle that did not happen is exactly what
the gate exists to refuse — kimi's framing, and it is the right one.
forgejo_leg_row folds all five assertions, and the leg no longer dispatches
after install or status has failed: a foreign runner's success could only ever
be evidence about someone else's runner. The regression composes the real
functions over a payload carrying a newer successful run, the way the leg
composes them.
The GitHub leg has the identical shape and is byte-frozen by #129's contract;
left alone deliberately, and worth a sibling issue for triage.
Refs #129
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-31 20:04:16 +00:00
for fn in tree_of assert_installed_from classify_leg capture_state emit_record forgejo_run_verdict forgejo_token_verdict forgejo_max_task_id forgejo_leg_row; do
2026-07-24 00:31:53 +00:00
check " extraction guards the awk: ${ fn } () landed " 0 " ${ fn } () { " grep -F " ${ fn } () { " " $FNS "
done
# shellcheck source=/dev/null
. " $FNS "
# =============================================================================
# tree_of — the versioned tree behind a CLI's symlink chain
# =============================================================================
IR = " $WORK /install " ; mkdir -p " $IR /versions/1.2.3/bin "
: > " $IR /versions/1.2.3/bin/rig "
ln -s "versions/1.2.3" " $IR /current "
mkdir -p " $WORK /bin "
ln -s " $IR /current/bin/rig " " $WORK /bin/rig "
check "tree_of resolves a current-symlink chain to versions/<v>" 0 " $IR /versions/1.2.3 " \
tree_of " $WORK /bin/rig "
ln -s " $IR /gone/bin/rig " " $WORK /bin/dangling "
check "tree_of refuses a dangling chain — a tree that is not there is not a tree" 1 "" \
tree_of " $WORK /bin/dangling "
# =============================================================================
# assert_installed_from — the up-front ref refusal, naming both refs
# =============================================================================
TREE = " $WORK /tree-main " ; mkdir -p " $TREE "
printf 'heavy-duty/rig@main\n' > " $TREE /INSTALLED_FROM "
check "matching INSTALLED_FROM passes silently" 0 "" \
assert_installed_from rig " $TREE " "heavy-duty/rig@main"
check "a mismatch refuses (the #103 hazard: asked release, got main)" 1 "FATAL" \
assert_installed_from rig " $TREE " "heavy-duty/rig@release/9.9.9"
check "…the refusal names the ref that was ASKED for" 1 "heavy-duty/rig@release/9.9.9" \
assert_installed_from rig " $TREE " "heavy-duty/rig@release/9.9.9"
check "…and the ref that actually LANDED" 1 "heavy-duty/rig@main" \
assert_installed_from rig " $TREE " "heavy-duty/rig@release/9.9.9"
check "an unreadable INSTALLED_FROM refuses too — absence is not a match" 1 "<unreadable>" \
assert_installed_from rig " $WORK /no-such-tree " "heavy-duty/rig@main"
# =============================================================================
# classify_leg — a loud skip is a SKIP, never a pass (box#153's defect class)
# =============================================================================
printf 'skip: docker not installed — nothing to exercise\n' > " $WORK /out-skip "
printf 'ok: seeded\nok: restored\n---\n14 passed, 0 failed\n' > " $WORK /out-pass "
printf 'FAIL: restore blew up\n' > " $WORK /out-fail "
check "exit 0 + 'skip:' line classifies as skip" 0 "skip" classify_leg 0 " $WORK /out-skip "
check "exit 0, no skip line, classifies as pass" 0 "pass" classify_leg 0 " $WORK /out-pass "
check "non-zero exit classifies as fail" 0 "fail" classify_leg 1 " $WORK /out-fail "
check "a skip line cannot rescue a non-zero exit (fail wins)" 0 "fail" \
classify_leg 1 " $WORK /out-skip "
# =============================================================================
# capture_state + diff — the idempotence verdict's machinery. The claim in
# #105's acceptance criteria: the assertion is a REAL diff of captured state,
# and it FAILS when convergence is broken — demonstrated here, mechanically,
# on every CI run, by breaking the state between two captures.
# =============================================================================
FIX = " $WORK /fix " ; mkdir -p " $FIX /sudoers.d "
printf 'role=staging-server root-door=open host=yes join=authkey\n' > " $FIX /role "
printf 'schema=1\nbootstrapped_by=9.9.9\nbootstrapped_at=T\nconverged_by=9.9.9\nconverged_at=T\n' > " $FIX /manifest "
printf 'dan active\nghost revoked\n' > " $FIX /ledger "
printf 'APT::Periodic::Update-Package-Lists "1";\n' > " $FIX /autoup "
printf '127.0.0.1 localhost\n127.0.1.1\tstaging-server\n' > " $FIX /hosts "
printf 'nosuchdrilluser ALL=(ALL) NOPASSWD:ALL\n' > " $FIX /sudoers.d/00-rig-nosuch "
# A stubbed sshd, so the effective-config section is exercised rather than
# skipped on a box with no daemon (repo precedent: test/release.sh's curl).
STUB = " $WORK /stub " ; mkdir -p " $STUB "
# The single-quoted $SSHD_FIXTURE is the STUB's expansion, not this shell's.
# shellcheck disable=SC2016
printf '#!/usr/bin/env bash\ncat "$SSHD_FIXTURE"\n' > " $STUB /sshd " ; chmod +x " $STUB /sshd "
printf 'passwordauthentication no\npermitrootlogin prohibit-password\n' > " $FIX /sshd-T "
cap( ) { # cap <outfile> — capture_state against the fixture set
RIG_ROLE_MARKER = " $FIX /role " RIG_MANIFEST = " $FIX /manifest " \
DRILL_LEDGER = " $FIX /ledger " DRILL_AUTOUPGRADES = " $FIX /autoup " \
DRILL_ETC_HOSTS = " $FIX /hosts " DRILL_SUDOERS_DIR = " $FIX /sudoers.d " \
SSHD_FIXTURE = " $FIX /sshd-T " PATH = " $STUB : $PATH " \
bash -c '. "$1"; capture_state "$2"' _ " $FNS " " $2 " 2>/dev/null
:
}
# cap runs capture_state in a child bash so the PATH stub cannot leak into
# this harness; $2 arrives as the capture's outfile.
cap out " $WORK /cap1 "
cap out " $WORK /cap2 "
check "two captures over untouched state diff EMPTY (the converged verdict)" 0 "" \
diff -u " $WORK /cap1 " " $WORK /cap2 "
check "the capture reads the fixtures, not the machine (marker line present)" 0 "role=staging-server" \
grep -o 'role=staging-server[^"]*' " $WORK /cap1 "
check "…the sshd section captured the effective config" 0 "passwordauthentication no" \
cat " $WORK /cap1 "
check "…a ledger user with no account reads as one, deterministically" 0 "(no account)" \
cat " $WORK /cap1 "
# Break convergence: the re-run "changed" the role marker and root's door.
printf 'role=staging-server root-door=closed host=yes join=authkey\n' > " $FIX /role "
printf 'passwordauthentication yes\npermitrootlogin prohibit-password\n' > " $FIX /sshd-T "
cap out " $WORK /cap3 "
check "a broken convergence makes the diff NON-empty — the assertion can fail" 1 "root-door=closed" \
diff -u " $WORK /cap1 " " $WORK /cap3 "
check "…and the diff names the drifted sshd keyword, not just 'differs'" 1 "passwordauthentication yes" \
diff -u " $WORK /cap1 " " $WORK /cap3 "
# =============================================================================
# emit_record — the record is drills/README.md's shape, and it cannot lie:
# a failed run still emits, a skipped leg is named, no clean-sweep reading.
# =============================================================================
emit( ) { # emit <outfile> — emit_record with the harness globals staged
DRILL_VERSION = "9.9.9" RUN_ID = "drill-2026-01-01-a" \
REF = "release/9.9.9" BOXREF = "release/0.4.0" RIG_SHA = "5d6e7f8" BOX_SHA = "1a2b3c4" \
2026-07-25 13:41:01 +00:00
TPLREPO = "heavy-duty/rig-templates" TPLREF = "9f8e7d6c5b4a39281706f5e4d3c2b1a098765432" TPL_SHA = "9f8e7d6" TPL_SOURCE = "snapshot" \
2026-07-24 00:31:53 +00:00
bash -c '
. " $1 "
pass = 12 fail = 1 skipped = 1
findings = ( "FAIL: coolify container state: absent" "SKIP: runner lifecycle: no --runner-repo fork given — the leg did not run" "NOTE: something worth a line" )
LEG_NAMES = ( "convergence — bootstrap staging-server reaches its role" "re-converge (idempotence)" "coolify install (4.1.2)" "runner lifecycle" )
LEG_RESULTS = ( "PASS (312s)" "clean, no changes" "FAIL — container absent" "SKIPPED — no fork provided" )
emit_record " $2 "
' _ " $FNS " " $2 "
}
emit out " $WORK /record.md "
check "record: the version-and-date heading" 0 "# Release drill — 9.9.9 — " head -1 " $WORK /record.md "
check "record: the run ID that joins the family's records" 0 "Run ID: drill-2026-01-01-a" cat " $WORK /record.md "
check "record: both pinned refs with their SHAs" 0 "rig@5d6e7f8 (RIG_REF=release/9.9.9)" cat " $WORK /record.md "
check "record: …box's too" 0 "box@1a2b3c4 (BOX_REF=release/0.4.0)" cat " $WORK /record.md "
2026-07-25 13:41:01 +00:00
check "record: the template registry SHA and actual source ride alongside the pair (#110/#153)" 0 "rig-templates@9f8e7d6 (ref 9f8e7d6c5b4a39281706f5e4d3c2b1a098765432, snapshot)" cat " $WORK /record.md "
2026-07-24 00:31:53 +00:00
check "record: one table row per leg, result verbatim" 0 "| re-converge (idempotence) | clean, no changes |" cat " $WORK /record.md "
check "record: the numbers, skips counted apart from passes" 0 "12 passed, 1 failed, 1 skipped" cat " $WORK /record.md "
check "record: a FAILED run still names what failed (evidence, not success)" 0 "FAIL: coolify container state: absent" cat " $WORK /record.md "
check "record: a skipped leg is stated as NOT run, by name" 0 "SKIP: runner lifecycle" cat " $WORK /record.md "
check "record: the skip section says the record is not evidence for it" 0 "not evidence" cat " $WORK /record.md "
check "record: the isolation boundary is named as box's, in words" 0 "NOT asserted here" cat " $WORK /record.md "
refute "record with a skip cannot read as a clean sweep" "Failed: nothing" " $WORK /record.md "
refute "notes are findings for the log, not failures for the record" "NOTE: something" " $WORK /record.md "
# The all-green shape: says so plainly, and only then.
DRILL_VERSION = "9.9.9" RUN_ID = "drill-2026-01-01-a" \
REF = "release/9.9.9" BOXREF = "release/0.4.0" RIG_SHA = "5d6e7f8" BOX_SHA = "1a2b3c4" \
bash -c '
. " $1 "
pass = 20 fail = 0 skipped = 0
findings = ( )
LEG_NAMES = ( "convergence" "re-converge (idempotence)" )
LEG_RESULTS = ( "PASS" "clean, no changes" )
emit_record " $2 "
' _ " $FNS " " $WORK /record-green.md "
check "an all-green record says every leg ran and passed" 0 "Every leg ran and every check passed" \
cat " $WORK /record-green.md "
# =============================================================================
# the shipped script itself
# =============================================================================
2026-07-30 23:27:36 +00:00
# =============================================================================
# forgejo_run_verdict — did OUR dispatched run land, and how (#129)
# =============================================================================
# The Forgejo half of the runner leg cannot reuse the GitHub reader. Measured
# against forgejo.heavyduty.builders (8.0.3+gitea-1.22.0) on 2026-07-30, a
# completed run in GET /repos/{o}/{r}/actions/tasks carries NO `conclusion`
# field at all — `status` holds the terminal outcome directly, where GitHub
# splits status:completed + conclusion:success. And `id` is a global task id
# (25) while the run's own URL ends in run_number (1), so the pre-dispatch
# guard has to compare `id`.
#
# The payload is also an ASSIGNED-task view: it reads total_count 0 for as
# long as a run sits queued (measured: 200s), so "no new id" is the ONLY
# signal that the runner never took the job. That is the verdict this leg
# exists to produce, which is why it gets its own function and its own tests.
FJ = " $WORK /fj " ; mkdir -p " $FJ "
printf '%s' '{"workflow_runs":[],"total_count":0}' > " $FJ /empty.json "
printf '%s' '{"workflow_runs":[{"id":25,"status":"success","run_number":1,"url":"https://f/o/r/actions/runs/1"}],"total_count":1}' > " $FJ /new-success.json "
printf '%s' '{"workflow_runs":[{"id":25,"status":"failure","run_number":1,"url":"https://f/o/r/actions/runs/1"}],"total_count":1}' > " $FJ /new-failure.json "
printf '%s' '{"workflow_runs":[{"id":25,"status":"cancelled","run_number":1,"url":"https://f/o/r/actions/runs/1"}],"total_count":1}' > " $FJ /new-cancelled.json "
printf '%s' '{"workflow_runs":[{"id":24,"status":"success","run_number":1,"url":"https://f/o/r/actions/runs/1"}],"total_count":1}' > " $FJ /stale-only.json "
check "verdict: an empty task list is PENDING, never a pass" 0 "pending" \
forgejo_run_verdict "" " $FJ /empty.json "
check "verdict: a queued run the runner never took stays PENDING" 0 "pending" \
forgejo_run_verdict "24" " $FJ /stale-only.json "
check "verdict: OUR new run, status success, is SUCCESS" 0 "success" \
forgejo_run_verdict "24" " $FJ /new-success.json "
check "verdict: status carries the outcome — failure is FAILED, not success" 0 "failed" \
forgejo_run_verdict "24" " $FJ /new-failure.json "
check "verdict: a cancelled run is FAILED, not silently passed" 0 "failed" \
forgejo_run_verdict "24" " $FJ /new-cancelled.json "
check "verdict: the first run ever (no pre-id) still resolves" 0 "success" \
forgejo_run_verdict "" " $FJ /new-success.json "
# A task appears in this payload the moment it is ASSIGNED, which can be before
# it finishes — so a non-terminal status must read as pending, not as a failure.
# Calling a still-running job "failed" would make the leg flaky in exactly the
# window the leg is watching.
printf '%s' '{"workflow_runs":[{"id":25,"status":"running","run_number":1}],"total_count":1}' > " $FJ /new-running.json "
check "verdict: an assigned-but-running task is PENDING, not FAILED" 0 "pending" \
forgejo_run_verdict "24" " $FJ /new-running.json "
2026-07-31 00:13:59 +00:00
# grok/kimi on !130: the reader must not stop at the FIRST run. actions/tasks
# accumulates — the moment a repo is drilled twice, our run shares the payload
# with older ones, and nothing documents the sort order. Reading entry[0] makes
# a green job read as a timeout, which is a FALSE FAILURE on the very gate this
# leg exists to provide.
printf '%s' '{"workflow_runs":[{"id":24,"status":"success"},{"id":25,"status":"success"}],"total_count":2}' > " $FJ /oldest-first.json "
printf '%s' '{"workflow_runs":[{"id":25,"status":"success"},{"id":24,"status":"success"}],"total_count":2}' > " $FJ /newest-first.json "
printf '%s' '{"workflow_runs":[{"id":25,"status":"running"},{"id":26,"status":"success"}],"total_count":2}' > " $FJ /ours-not-first.json "
printf '%s' '{"workflow_runs":[{"id":23,"status":"success"},{"id":24,"status":"failure"}],"total_count":2}' > " $FJ /all-stale.json "
# Pretty-printed: the instance may or may not compact its JSON, and a parser
# that silently depends on one-line objects is a latent failure (kimi, !130).
2026-07-31 00:29:06 +00:00
# Written HERE, like every other fixture: a suite that copies from a scratch
# path passes only on the box that built it (grok/kimi, !130 round 2).
printf '%s\n' ' {
"workflow_runs" : [
{ "id" : 24, "status" : "success" } ,
{ "id" : 25, "name" : "drill" , "status" : "success" }
] ,
"total_count" : 2
} ' > " $FJ /pretty.json "
2026-07-31 00:13:59 +00:00
check "verdict: ours is LAST in the payload — order must not decide" 0 "success" \
forgejo_run_verdict "24" " $FJ /oldest-first.json "
check "verdict: ours is FIRST in the payload — same answer" 0 "success" \
forgejo_run_verdict "24" " $FJ /newest-first.json "
check "verdict: a stale RUNNING entry ahead of ours does not mask it" 0 "success" \
forgejo_run_verdict "24" " $FJ /ours-not-first.json "
check "verdict: every entry at or below pre is stale — PENDING" 0 "pending" \
forgejo_run_verdict "24" " $FJ /all-stale.json "
check "verdict: a pretty-printed payload parses too" 0 "success" \
forgejo_run_verdict "24" " $FJ /pretty.json "
2026-07-31 00:29:06 +00:00
# The PRE-DISPATCH snapshot has the same multi-entry hazard as the verdict, and
# getting it wrong is worse: a `head -n1` pre-id on an oldest-first payload
# names an OLD run as the baseline, so a later poll that finds the same body
# reports the PREVIOUS drill's run as ours — a false PASS on the take-a-job
# assertion, where the entry[0] bug only produced a false failure (grok, !130).
# Both sides must fold max over every id, which is why they share one function.
check "max id: oldest-first payload yields the NEWEST id, not the first" 0 "25" \
forgejo_max_task_id " $FJ /oldest-first.json "
check "max id: newest-first payload yields the same answer" 0 "25" \
forgejo_max_task_id " $FJ /newest-first.json "
check "max id: an empty payload has no id at all" 0 "" \
forgejo_max_task_id " $FJ /empty.json "
check "max id: a pretty-printed payload folds too" 0 "25" \
forgejo_max_task_id " $FJ /pretty.json "
# The false PASS, pinned end to end: snapshot the oldest-first body, dispatch,
# the runner never takes it so the body is unchanged — the verdict must stay
# pending. With head -n1 this returned success.
# The two halves composed exactly as the leg composes them.
verdict_after_no_new_run( ) { forgejo_run_verdict " $( forgejo_max_task_id " $1 " ) " " $1 " ; }
check "no new run after dispatch: max-id baseline keeps it PENDING (false-PASS guard)" 0 "pending" \
verdict_after_no_new_run " $FJ /oldest-first.json "
check "…and the same composition on a pretty payload" 0 "pending" \
verdict_after_no_new_run " $FJ /pretty.json "
fix(drill): the leg row is the whole lifecycle, not just the job
codex, grok and kimi all landed on the same hole at e42ba31, and they are
right. The record row keyed only on fj_took, so install/status/remove failures
called `no` without gating it. A dispatched job only needs SOMETHING answering
runs-on: drill — and this leg removes locally, telling the operator to delete
the stale runner by hand, so a leftover drill-labeled runner from the previous
drill is the designed-for aftermath, not a contrived case. It answers the
fixture exactly, and the row then read "PASS — registered, took a job,
removed" for a drill whose install never happened.
The exit code was never wrong: every one of those failures also called `no`.
What was wrong is the row, and the row is what outlives the run in
drills/<v>.md. A row claiming a lifecycle that did not happen is exactly what
the gate exists to refuse — kimi's framing, and it is the right one.
forgejo_leg_row folds all five assertions, and the leg no longer dispatches
after install or status has failed: a foreign runner's success could only ever
be evidence about someone else's runner. The regression composes the real
functions over a payload carrying a newer successful run, the way the leg
composes them.
The GitHub leg has the identical shape and is byte-frozen by #129's contract;
left alone deliberately, and worth a sibling issue for triage.
Refs #129
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-31 20:04:16 +00:00
# =============================================================================
# forgejo_leg_row — the row is the WHOLE lifecycle, not just the job
# =============================================================================
# codex/grok/kimi on !130: keying the record row on the take-a-job outcome alone
# lets it read "PASS — registered, took a job, removed" when install failed, so
# long as SOMETHING answered runs-on: drill. That is not contrived — this leg
# removes locally and tells the operator to delete the stale runner by hand, so
# a leftover drill-labeled runner from the previous drill is the DESIGNED-FOR
# aftermath, and it answers the fixture exactly.
#
# drills/<v>.md is the release's durable evidence. A row claiming a lifecycle
# that did not happen is precisely what the gate exists to refuse, so PASS
# requires every assertion, not just the interesting one.
check "leg row: everything succeeded is the only PASS" 0 "PASS" \
forgejo_leg_row 1 1 success 1 1
check "leg row: install failed cannot PASS, even when a foreign runner took the job" 0 "FAIL" \
forgejo_leg_row 0 1 success 1 1
check "leg row: status failed cannot PASS either" 0 "FAIL" \
forgejo_leg_row 1 0 success 1 1
check "leg row: remove failed cannot PASS" 0 "FAIL" \
forgejo_leg_row 1 1 success 0 1
check "leg row: a runner still registered after remove cannot PASS" 0 "FAIL" \
forgejo_leg_row 1 1 success 1 0
check "leg row: no dispatch attempted, everything else clean, is PARTIAL" 0 "PARTIAL" \
forgejo_leg_row 1 1 none 1 1
check "leg row: a job that was never taken is a FAIL" 0 "FAIL" \
forgejo_leg_row 1 1 timeout 1 1
check "leg row: PARTIAL requires a clean lifecycle too" 0 "FAIL" \
forgejo_leg_row 0 1 none 1 1
# The end-to-end shape codex/grok/kimi asked for, composed the way the leg
# composes it: a tasks payload carrying a NEWER successful run (as a foreign
# drill-labeled runner would produce) must still not yield a PASS row when the
# drill's own install failed. This is the exact false-evidence case.
row_after_failed_install( ) {
forgejo_leg_row 0 1 " $( forgejo_run_verdict " $( forgejo_max_task_id " $1 " ) " " $1 " ) " 1 1
}
printf '%s' '{"workflow_runs":[{"id":24,"status":"success"},{"id":99,"status":"success"}],"total_count":2}' \
> " $FJ /foreign-runner-took-it.json "
check "install failed + a newer successful run in the payload is still FAIL, never PASS" 0 "FAIL" \
row_after_failed_install " $FJ /foreign-runner-took-it.json "
# …and the same payload with a clean lifecycle is the PASS, so the check above
# is discriminating rather than always-FAIL.
row_after_clean_install( ) {
forgejo_leg_row 1 1 " $( forgejo_run_verdict "" " $1 " ) " 1 1
}
check "…while the same payload with a clean lifecycle does PASS" 0 "PASS" \
row_after_clean_install " $FJ /foreign-runner-took-it.json "
2026-07-31 00:13:59 +00:00
# =============================================================================
# forgejo_token_verdict — a configured leg that cannot mint must FAIL, not SKIP
# =============================================================================
# #129's own acceptance: "Token source present but the instance is unreachable
# -> the leg FAILS; it must not skip and must not pass". A mint that returns
# nothing because the instance is unreachable, the token is under-scoped or the
# repo name is wrong is a CONFIGURED leg failing — reporting "no token source"
# sends the operator to check an env var they already set, and writes SKIPPED
# where the record owes a FAIL. That is the UNREADABLE-vs-NONE shape
# drills/README.md names.
check "token: a resolved registration token is ok" 0 "ok" \
forgejo_token_verdict "reg-tok" ""
check "token: an explicit token wins even with no API token" 0 "ok" \
forgejo_token_verdict "reg-tok" ""
check "token: no token at all and no API token is a genuine SKIP" 0 "no-source" \
forgejo_token_verdict "" ""
check "token: API token offered but mint produced nothing is a FAILURE" 0 "mint-failed" \
forgejo_token_verdict "" "api-tok"
2026-07-30 23:27:36 +00:00
# The anti-false-positive guard, stated as its own case: an OLD completed run
# with the SAME id as pre_id must never be read as this dispatch's result.
check "verdict: a pre-existing success with the pre-id is NOT our run" 0 "pending" \
forgejo_run_verdict "25" " $FJ /new-success.json "
2026-07-24 00:31:53 +00:00
# Arg refusals fire before the root check (repo doctrine, bootstrap.sh:114),
# which is what makes them provable here without a throwaway machine.
check "drill.sh refuses to run without BOTH refs pinned (#103)" 2 "--box-ref" \
env -u RIG_REF -u BOX_REF bash " $ROOT /drill/drill.sh " --rig-ref release/9.9.9 --yes
check "…and the refusal shows which ref is missing" 2 "<unset>" \
env -u RIG_REF -u BOX_REF bash " $ROOT /drill/drill.sh " --rig-ref release/9.9.9 --yes
check "a tenant role is refused — the drill converges machines, not guests" 2 "not a machine role" \
bash " $ROOT /drill/drill.sh " --rig-ref r --box-ref b --role claude-box --yes
check "no --users is a refusal, naming why the drill will not default it" 2 "--users <path> is required" \
bash " $ROOT /drill/drill.sh " --rig-ref r --box-ref b --yes
check "an unreadable users file dies before anything is spent" 2 "cannot read users file" \
bash " $ROOT /drill/drill.sh " --rig-ref r --box-ref b --users " $WORK /no-such-users " --yes
check "an unknown flag dies loudly, exit 2" 2 "unknown option" \
bash " $ROOT /drill/drill.sh " --frobnicate
check "--help prints the header and exits 0" 0 "THROWAWAY" \
bash " $ROOT /drill/drill.sh " --help
2026-07-30 23:27:36 +00:00
check "--forgejo-instance is a known flag (the leg's opt-in)" 2 "--users <path> is required" \
bash " $ROOT /drill/drill.sh " --rig-ref r --box-ref b --forgejo-instance https://f.example.com --yes
check "--forgejo-runner-repo is a known flag" 2 "--users <path> is required" \
bash " $ROOT /drill/drill.sh " --rig-ref r --box-ref b --forgejo-runner-repo o/r --yes
check "--help names the forgejo runner leg's flags" 0 "--forgejo-instance" \
bash " $ROOT /drill/drill.sh " --help
check "--forgejo-ref is a known flag (Forgejo's dispatch needs a ref)" 2 "--users <path> is required" \
bash " $ROOT /drill/drill.sh " --rig-ref r --box-ref b --forgejo-ref dev --yes
2026-07-24 00:31:53 +00:00
echo "---"
echo " $PASS passed, $FAIL failed "
[ " $FAIL " -eq 0 ]