cluade-reviewer-andresmgsl
9afcdf858f
codex, grok and kimi all landed on the same hole at e42ba31, and they are
right. The record row keyed only on fj_took, so install/status/remove failures
called `no` without gating it. A dispatched job only needs SOMETHING answering
runs-on: drill — and this leg removes locally, telling the operator to delete
the stale runner by hand, so a leftover drill-labeled runner from the previous
drill is the designed-for aftermath, not a contrived case. It answers the
fixture exactly, and the row then read "PASS — registered, took a job,
removed" for a drill whose install never happened.
The exit code was never wrong: every one of those failures also called `no`.
What was wrong is the row, and the row is what outlives the run in
drills/<v>.md. A row claiming a lifecycle that did not happen is exactly what
the gate exists to refuse — kimi's framing, and it is the right one.
forgejo_leg_row folds all five assertions, and the leg no longer dispatches
after install or status has failed: a foreign runner's success could only ever
be evidence about someone else's runner. The regression composes the real
functions over a payload carrying a newer successful run, the way the leg
composes them.
The GitHub leg has the identical shape and is byte-frozen by #129's contract;
left alone deliberately, and worth a sibling issue for triage.
Refs #129
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
402 lines
24 KiB
Bash
402 lines
24 KiB
Bash
#!/usr/bin/env bash
|
|
# test/drill.sh — the drill harness's HONESTY, proven without hardware.
|
|
#
|
|
# drill/drill.sh is the instrument (#105), so what this suite tests is the
|
|
# instrument itself: the refusals, the classifications, the capture-and-diff
|
|
# that decides idempotence, and the record emitter — the parts whose lies
|
|
# would be believed, months later, by a reader of drills/<version>.md. The
|
|
# four-leg live run on a real Debian machine is #107's exercise, not this
|
|
# file's: nothing here needs root, Docker, a tailnet or the network.
|
|
#
|
|
# Extraction pattern is test/release.sh's: the functions under test are
|
|
# awk-extracted from drill/drill.sh and driven against fixtures, so the tests
|
|
# exercise the shipped bytes, and the extraction check itself guards the awk
|
|
# against a drifted function boundary.
|
|
# Deliberately no `set -e` — the harness asserts on failing commands.
|
|
set -u
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "$ROOT" || exit 1
|
|
PASS=0 FAIL=0
|
|
|
|
# check <desc> <want_exit> <want_substr> <cmd...>
|
|
check() {
|
|
local desc="$1" want="$2" substr="$3"; shift 3
|
|
local out rc
|
|
out="$("$@" 2>&1)"; rc=$?
|
|
if [ "$rc" -ne "$want" ]; then
|
|
echo "FAIL: $desc — exit $rc, wanted $want"
|
|
printf '%s\n' "$out" | sed 's/^/ /'
|
|
FAIL=$((FAIL + 1)); return
|
|
fi
|
|
if [ -n "$substr" ] && ! printf '%s' "$out" | grep -qF -e "$substr"; then
|
|
echo "FAIL: $desc — output missing '$substr'"
|
|
printf '%s\n' "$out" | sed 's/^/ /'
|
|
FAIL=$((FAIL + 1)); return
|
|
fi
|
|
echo "ok: $desc"; PASS=$((PASS + 1))
|
|
}
|
|
|
|
# refute <desc> <substr> <file> — the file must NOT contain the substring.
|
|
refute() {
|
|
if grep -qF -e "$2" "$3"; then
|
|
echo "FAIL: $1 — found forbidden '$2'"
|
|
FAIL=$((FAIL + 1)); return
|
|
fi
|
|
echo "ok: $1"; PASS=$((PASS + 1))
|
|
}
|
|
|
|
WORK="$(mktemp -d)"
|
|
trap 'rm -rf "$WORK"' EXIT
|
|
|
|
# --- the functions under test, extracted -------------------------------------
|
|
FNS="$WORK/drill-fns.sh"
|
|
for fn in tree_of assert_installed_from classify_leg capture_state emit_record forgejo_run_verdict forgejo_token_verdict forgejo_max_task_id forgejo_leg_row; do
|
|
awk "/^${fn}\(\) \{/,/^\}/" "$ROOT/drill/drill.sh" >> "$FNS"
|
|
done
|
|
for fn in tree_of assert_installed_from classify_leg capture_state emit_record forgejo_run_verdict forgejo_token_verdict forgejo_max_task_id forgejo_leg_row; do
|
|
check "extraction guards the awk: ${fn}() landed" 0 "${fn}() {" grep -F "${fn}() {" "$FNS"
|
|
done
|
|
# shellcheck source=/dev/null
|
|
. "$FNS"
|
|
|
|
# =============================================================================
|
|
# tree_of — the versioned tree behind a CLI's symlink chain
|
|
# =============================================================================
|
|
IR="$WORK/install"; mkdir -p "$IR/versions/1.2.3/bin"
|
|
: > "$IR/versions/1.2.3/bin/rig"
|
|
ln -s "versions/1.2.3" "$IR/current"
|
|
mkdir -p "$WORK/bin"
|
|
ln -s "$IR/current/bin/rig" "$WORK/bin/rig"
|
|
check "tree_of resolves a current-symlink chain to versions/<v>" 0 "$IR/versions/1.2.3" \
|
|
tree_of "$WORK/bin/rig"
|
|
ln -s "$IR/gone/bin/rig" "$WORK/bin/dangling"
|
|
check "tree_of refuses a dangling chain — a tree that is not there is not a tree" 1 "" \
|
|
tree_of "$WORK/bin/dangling"
|
|
|
|
# =============================================================================
|
|
# assert_installed_from — the up-front ref refusal, naming both refs
|
|
# =============================================================================
|
|
TREE="$WORK/tree-main"; mkdir -p "$TREE"
|
|
printf 'heavy-duty/rig@main\n' > "$TREE/INSTALLED_FROM"
|
|
check "matching INSTALLED_FROM passes silently" 0 "" \
|
|
assert_installed_from rig "$TREE" "heavy-duty/rig@main"
|
|
check "a mismatch refuses (the #103 hazard: asked release, got main)" 1 "FATAL" \
|
|
assert_installed_from rig "$TREE" "heavy-duty/rig@release/9.9.9"
|
|
check "…the refusal names the ref that was ASKED for" 1 "heavy-duty/rig@release/9.9.9" \
|
|
assert_installed_from rig "$TREE" "heavy-duty/rig@release/9.9.9"
|
|
check "…and the ref that actually LANDED" 1 "heavy-duty/rig@main" \
|
|
assert_installed_from rig "$TREE" "heavy-duty/rig@release/9.9.9"
|
|
check "an unreadable INSTALLED_FROM refuses too — absence is not a match" 1 "<unreadable>" \
|
|
assert_installed_from rig "$WORK/no-such-tree" "heavy-duty/rig@main"
|
|
|
|
# =============================================================================
|
|
# classify_leg — a loud skip is a SKIP, never a pass (box#153's defect class)
|
|
# =============================================================================
|
|
printf 'skip: docker not installed — nothing to exercise\n' > "$WORK/out-skip"
|
|
printf 'ok: seeded\nok: restored\n---\n14 passed, 0 failed\n' > "$WORK/out-pass"
|
|
printf 'FAIL: restore blew up\n' > "$WORK/out-fail"
|
|
check "exit 0 + 'skip:' line classifies as skip" 0 "skip" classify_leg 0 "$WORK/out-skip"
|
|
check "exit 0, no skip line, classifies as pass" 0 "pass" classify_leg 0 "$WORK/out-pass"
|
|
check "non-zero exit classifies as fail" 0 "fail" classify_leg 1 "$WORK/out-fail"
|
|
check "a skip line cannot rescue a non-zero exit (fail wins)" 0 "fail" \
|
|
classify_leg 1 "$WORK/out-skip"
|
|
|
|
# =============================================================================
|
|
# capture_state + diff — the idempotence verdict's machinery. The claim in
|
|
# #105's acceptance criteria: the assertion is a REAL diff of captured state,
|
|
# and it FAILS when convergence is broken — demonstrated here, mechanically,
|
|
# on every CI run, by breaking the state between two captures.
|
|
# =============================================================================
|
|
FIX="$WORK/fix"; mkdir -p "$FIX/sudoers.d"
|
|
printf 'role=staging-server root-door=open host=yes join=authkey\n' > "$FIX/role"
|
|
printf 'schema=1\nbootstrapped_by=9.9.9\nbootstrapped_at=T\nconverged_by=9.9.9\nconverged_at=T\n' > "$FIX/manifest"
|
|
printf 'dan active\nghost revoked\n' > "$FIX/ledger"
|
|
printf 'APT::Periodic::Update-Package-Lists "1";\n' > "$FIX/autoup"
|
|
printf '127.0.0.1 localhost\n127.0.1.1\tstaging-server\n' > "$FIX/hosts"
|
|
printf 'nosuchdrilluser ALL=(ALL) NOPASSWD:ALL\n' > "$FIX/sudoers.d/00-rig-nosuch"
|
|
# A stubbed sshd, so the effective-config section is exercised rather than
|
|
# skipped on a box with no daemon (repo precedent: test/release.sh's curl).
|
|
STUB="$WORK/stub"; mkdir -p "$STUB"
|
|
# The single-quoted $SSHD_FIXTURE is the STUB's expansion, not this shell's.
|
|
# shellcheck disable=SC2016
|
|
printf '#!/usr/bin/env bash\ncat "$SSHD_FIXTURE"\n' > "$STUB/sshd"; chmod +x "$STUB/sshd"
|
|
printf 'passwordauthentication no\npermitrootlogin prohibit-password\n' > "$FIX/sshd-T"
|
|
|
|
cap() { # cap <outfile> — capture_state against the fixture set
|
|
RIG_ROLE_MARKER="$FIX/role" RIG_MANIFEST="$FIX/manifest" \
|
|
DRILL_LEDGER="$FIX/ledger" DRILL_AUTOUPGRADES="$FIX/autoup" \
|
|
DRILL_ETC_HOSTS="$FIX/hosts" DRILL_SUDOERS_DIR="$FIX/sudoers.d" \
|
|
SSHD_FIXTURE="$FIX/sshd-T" PATH="$STUB:$PATH" \
|
|
bash -c '. "$1"; capture_state "$2"' _ "$FNS" "$2" 2>/dev/null
|
|
:
|
|
}
|
|
# cap runs capture_state in a child bash so the PATH stub cannot leak into
|
|
# this harness; $2 arrives as the capture's outfile.
|
|
cap out "$WORK/cap1"
|
|
cap out "$WORK/cap2"
|
|
check "two captures over untouched state diff EMPTY (the converged verdict)" 0 "" \
|
|
diff -u "$WORK/cap1" "$WORK/cap2"
|
|
check "the capture reads the fixtures, not the machine (marker line present)" 0 "role=staging-server" \
|
|
grep -o 'role=staging-server[^"]*' "$WORK/cap1"
|
|
check "…the sshd section captured the effective config" 0 "passwordauthentication no" \
|
|
cat "$WORK/cap1"
|
|
check "…a ledger user with no account reads as one, deterministically" 0 "(no account)" \
|
|
cat "$WORK/cap1"
|
|
|
|
# Break convergence: the re-run "changed" the role marker and root's door.
|
|
printf 'role=staging-server root-door=closed host=yes join=authkey\n' > "$FIX/role"
|
|
printf 'passwordauthentication yes\npermitrootlogin prohibit-password\n' > "$FIX/sshd-T"
|
|
cap out "$WORK/cap3"
|
|
check "a broken convergence makes the diff NON-empty — the assertion can fail" 1 "root-door=closed" \
|
|
diff -u "$WORK/cap1" "$WORK/cap3"
|
|
check "…and the diff names the drifted sshd keyword, not just 'differs'" 1 "passwordauthentication yes" \
|
|
diff -u "$WORK/cap1" "$WORK/cap3"
|
|
|
|
# =============================================================================
|
|
# emit_record — the record is drills/README.md's shape, and it cannot lie:
|
|
# a failed run still emits, a skipped leg is named, no clean-sweep reading.
|
|
# =============================================================================
|
|
emit() { # emit <outfile> — emit_record with the harness globals staged
|
|
DRILL_VERSION="9.9.9" RUN_ID="drill-2026-01-01-a" \
|
|
REF="release/9.9.9" BOXREF="release/0.4.0" RIG_SHA="5d6e7f8" BOX_SHA="1a2b3c4" \
|
|
TPLREPO="heavy-duty/rig-templates" TPLREF="9f8e7d6c5b4a39281706f5e4d3c2b1a098765432" TPL_SHA="9f8e7d6" TPL_SOURCE="snapshot" \
|
|
bash -c '
|
|
. "$1"
|
|
pass=12 fail=1 skipped=1
|
|
findings=("FAIL: coolify container state: absent" "SKIP: runner lifecycle: no --runner-repo fork given — the leg did not run" "NOTE: something worth a line")
|
|
LEG_NAMES=("convergence — bootstrap staging-server reaches its role" "re-converge (idempotence)" "coolify install (4.1.2)" "runner lifecycle")
|
|
LEG_RESULTS=("PASS (312s)" "clean, no changes" "FAIL — container absent" "SKIPPED — no fork provided")
|
|
emit_record "$2"
|
|
' _ "$FNS" "$2"
|
|
}
|
|
emit out "$WORK/record.md"
|
|
check "record: the version-and-date heading" 0 "# Release drill — 9.9.9 — " head -1 "$WORK/record.md"
|
|
check "record: the run ID that joins the family's records" 0 "Run ID: drill-2026-01-01-a" cat "$WORK/record.md"
|
|
check "record: both pinned refs with their SHAs" 0 "rig@5d6e7f8 (RIG_REF=release/9.9.9)" cat "$WORK/record.md"
|
|
check "record: …box's too" 0 "box@1a2b3c4 (BOX_REF=release/0.4.0)" cat "$WORK/record.md"
|
|
check "record: the template registry SHA and actual source ride alongside the pair (#110/#153)" 0 "rig-templates@9f8e7d6 (ref 9f8e7d6c5b4a39281706f5e4d3c2b1a098765432, snapshot)" cat "$WORK/record.md"
|
|
check "record: one table row per leg, result verbatim" 0 "| re-converge (idempotence) | clean, no changes |" cat "$WORK/record.md"
|
|
check "record: the numbers, skips counted apart from passes" 0 "12 passed, 1 failed, 1 skipped" cat "$WORK/record.md"
|
|
check "record: a FAILED run still names what failed (evidence, not success)" 0 "FAIL: coolify container state: absent" cat "$WORK/record.md"
|
|
check "record: a skipped leg is stated as NOT run, by name" 0 "SKIP: runner lifecycle" cat "$WORK/record.md"
|
|
check "record: the skip section says the record is not evidence for it" 0 "not evidence" cat "$WORK/record.md"
|
|
check "record: the isolation boundary is named as box's, in words" 0 "NOT asserted here" cat "$WORK/record.md"
|
|
refute "record with a skip cannot read as a clean sweep" "Failed: nothing" "$WORK/record.md"
|
|
refute "notes are findings for the log, not failures for the record" "NOTE: something" "$WORK/record.md"
|
|
|
|
# The all-green shape: says so plainly, and only then.
|
|
DRILL_VERSION="9.9.9" RUN_ID="drill-2026-01-01-a" \
|
|
REF="release/9.9.9" BOXREF="release/0.4.0" RIG_SHA="5d6e7f8" BOX_SHA="1a2b3c4" \
|
|
bash -c '
|
|
. "$1"
|
|
pass=20 fail=0 skipped=0
|
|
findings=()
|
|
LEG_NAMES=("convergence" "re-converge (idempotence)")
|
|
LEG_RESULTS=("PASS" "clean, no changes")
|
|
emit_record "$2"
|
|
' _ "$FNS" "$WORK/record-green.md"
|
|
check "an all-green record says every leg ran and passed" 0 "Every leg ran and every check passed" \
|
|
cat "$WORK/record-green.md"
|
|
|
|
# =============================================================================
|
|
# the shipped script itself
|
|
# =============================================================================
|
|
# =============================================================================
|
|
# forgejo_run_verdict — did OUR dispatched run land, and how (#129)
|
|
# =============================================================================
|
|
# The Forgejo half of the runner leg cannot reuse the GitHub reader. Measured
|
|
# against forgejo.heavyduty.builders (8.0.3+gitea-1.22.0) on 2026-07-30, a
|
|
# completed run in GET /repos/{o}/{r}/actions/tasks carries NO `conclusion`
|
|
# field at all — `status` holds the terminal outcome directly, where GitHub
|
|
# splits status:completed + conclusion:success. And `id` is a global task id
|
|
# (25) while the run's own URL ends in run_number (1), so the pre-dispatch
|
|
# guard has to compare `id`.
|
|
#
|
|
# The payload is also an ASSIGNED-task view: it reads total_count 0 for as
|
|
# long as a run sits queued (measured: 200s), so "no new id" is the ONLY
|
|
# signal that the runner never took the job. That is the verdict this leg
|
|
# exists to produce, which is why it gets its own function and its own tests.
|
|
FJ="$WORK/fj"; mkdir -p "$FJ"
|
|
printf '%s' '{"workflow_runs":[],"total_count":0}' > "$FJ/empty.json"
|
|
printf '%s' '{"workflow_runs":[{"id":25,"status":"success","run_number":1,"url":"https://f/o/r/actions/runs/1"}],"total_count":1}' > "$FJ/new-success.json"
|
|
printf '%s' '{"workflow_runs":[{"id":25,"status":"failure","run_number":1,"url":"https://f/o/r/actions/runs/1"}],"total_count":1}' > "$FJ/new-failure.json"
|
|
printf '%s' '{"workflow_runs":[{"id":25,"status":"cancelled","run_number":1,"url":"https://f/o/r/actions/runs/1"}],"total_count":1}' > "$FJ/new-cancelled.json"
|
|
printf '%s' '{"workflow_runs":[{"id":24,"status":"success","run_number":1,"url":"https://f/o/r/actions/runs/1"}],"total_count":1}' > "$FJ/stale-only.json"
|
|
|
|
check "verdict: an empty task list is PENDING, never a pass" 0 "pending" \
|
|
forgejo_run_verdict "" "$FJ/empty.json"
|
|
check "verdict: a queued run the runner never took stays PENDING" 0 "pending" \
|
|
forgejo_run_verdict "24" "$FJ/stale-only.json"
|
|
check "verdict: OUR new run, status success, is SUCCESS" 0 "success" \
|
|
forgejo_run_verdict "24" "$FJ/new-success.json"
|
|
check "verdict: status carries the outcome — failure is FAILED, not success" 0 "failed" \
|
|
forgejo_run_verdict "24" "$FJ/new-failure.json"
|
|
check "verdict: a cancelled run is FAILED, not silently passed" 0 "failed" \
|
|
forgejo_run_verdict "24" "$FJ/new-cancelled.json"
|
|
check "verdict: the first run ever (no pre-id) still resolves" 0 "success" \
|
|
forgejo_run_verdict "" "$FJ/new-success.json"
|
|
# A task appears in this payload the moment it is ASSIGNED, which can be before
|
|
# it finishes — so a non-terminal status must read as pending, not as a failure.
|
|
# Calling a still-running job "failed" would make the leg flaky in exactly the
|
|
# window the leg is watching.
|
|
printf '%s' '{"workflow_runs":[{"id":25,"status":"running","run_number":1}],"total_count":1}' > "$FJ/new-running.json"
|
|
check "verdict: an assigned-but-running task is PENDING, not FAILED" 0 "pending" \
|
|
forgejo_run_verdict "24" "$FJ/new-running.json"
|
|
|
|
# grok/kimi on !130: the reader must not stop at the FIRST run. actions/tasks
|
|
# accumulates — the moment a repo is drilled twice, our run shares the payload
|
|
# with older ones, and nothing documents the sort order. Reading entry[0] makes
|
|
# a green job read as a timeout, which is a FALSE FAILURE on the very gate this
|
|
# leg exists to provide.
|
|
printf '%s' '{"workflow_runs":[{"id":24,"status":"success"},{"id":25,"status":"success"}],"total_count":2}' > "$FJ/oldest-first.json"
|
|
printf '%s' '{"workflow_runs":[{"id":25,"status":"success"},{"id":24,"status":"success"}],"total_count":2}' > "$FJ/newest-first.json"
|
|
printf '%s' '{"workflow_runs":[{"id":25,"status":"running"},{"id":26,"status":"success"}],"total_count":2}' > "$FJ/ours-not-first.json"
|
|
printf '%s' '{"workflow_runs":[{"id":23,"status":"success"},{"id":24,"status":"failure"}],"total_count":2}' > "$FJ/all-stale.json"
|
|
# Pretty-printed: the instance may or may not compact its JSON, and a parser
|
|
# that silently depends on one-line objects is a latent failure (kimi, !130).
|
|
# Written HERE, like every other fixture: a suite that copies from a scratch
|
|
# path passes only on the box that built it (grok/kimi, !130 round 2).
|
|
printf '%s\n' '{
|
|
"workflow_runs": [
|
|
{"id": 24, "status": "success"},
|
|
{"id": 25, "name": "drill", "status": "success"}
|
|
],
|
|
"total_count": 2
|
|
}' > "$FJ/pretty.json"
|
|
check "verdict: ours is LAST in the payload — order must not decide" 0 "success" \
|
|
forgejo_run_verdict "24" "$FJ/oldest-first.json"
|
|
check "verdict: ours is FIRST in the payload — same answer" 0 "success" \
|
|
forgejo_run_verdict "24" "$FJ/newest-first.json"
|
|
check "verdict: a stale RUNNING entry ahead of ours does not mask it" 0 "success" \
|
|
forgejo_run_verdict "24" "$FJ/ours-not-first.json"
|
|
check "verdict: every entry at or below pre is stale — PENDING" 0 "pending" \
|
|
forgejo_run_verdict "24" "$FJ/all-stale.json"
|
|
check "verdict: a pretty-printed payload parses too" 0 "success" \
|
|
forgejo_run_verdict "24" "$FJ/pretty.json"
|
|
|
|
# The PRE-DISPATCH snapshot has the same multi-entry hazard as the verdict, and
|
|
# getting it wrong is worse: a `head -n1` pre-id on an oldest-first payload
|
|
# names an OLD run as the baseline, so a later poll that finds the same body
|
|
# reports the PREVIOUS drill's run as ours — a false PASS on the take-a-job
|
|
# assertion, where the entry[0] bug only produced a false failure (grok, !130).
|
|
# Both sides must fold max over every id, which is why they share one function.
|
|
check "max id: oldest-first payload yields the NEWEST id, not the first" 0 "25" \
|
|
forgejo_max_task_id "$FJ/oldest-first.json"
|
|
check "max id: newest-first payload yields the same answer" 0 "25" \
|
|
forgejo_max_task_id "$FJ/newest-first.json"
|
|
check "max id: an empty payload has no id at all" 0 "" \
|
|
forgejo_max_task_id "$FJ/empty.json"
|
|
check "max id: a pretty-printed payload folds too" 0 "25" \
|
|
forgejo_max_task_id "$FJ/pretty.json"
|
|
# The false PASS, pinned end to end: snapshot the oldest-first body, dispatch,
|
|
# the runner never takes it so the body is unchanged — the verdict must stay
|
|
# pending. With head -n1 this returned success.
|
|
# The two halves composed exactly as the leg composes them.
|
|
verdict_after_no_new_run() { forgejo_run_verdict "$(forgejo_max_task_id "$1")" "$1"; }
|
|
check "no new run after dispatch: max-id baseline keeps it PENDING (false-PASS guard)" 0 "pending" \
|
|
verdict_after_no_new_run "$FJ/oldest-first.json"
|
|
check "…and the same composition on a pretty payload" 0 "pending" \
|
|
verdict_after_no_new_run "$FJ/pretty.json"
|
|
|
|
# =============================================================================
|
|
# forgejo_leg_row — the row is the WHOLE lifecycle, not just the job
|
|
# =============================================================================
|
|
# codex/grok/kimi on !130: keying the record row on the take-a-job outcome alone
|
|
# lets it read "PASS — registered, took a job, removed" when install failed, so
|
|
# long as SOMETHING answered runs-on: drill. That is not contrived — this leg
|
|
# removes locally and tells the operator to delete the stale runner by hand, so
|
|
# a leftover drill-labeled runner from the previous drill is the DESIGNED-FOR
|
|
# aftermath, and it answers the fixture exactly.
|
|
#
|
|
# drills/<v>.md is the release's durable evidence. A row claiming a lifecycle
|
|
# that did not happen is precisely what the gate exists to refuse, so PASS
|
|
# requires every assertion, not just the interesting one.
|
|
check "leg row: everything succeeded is the only PASS" 0 "PASS" \
|
|
forgejo_leg_row 1 1 success 1 1
|
|
check "leg row: install failed cannot PASS, even when a foreign runner took the job" 0 "FAIL" \
|
|
forgejo_leg_row 0 1 success 1 1
|
|
check "leg row: status failed cannot PASS either" 0 "FAIL" \
|
|
forgejo_leg_row 1 0 success 1 1
|
|
check "leg row: remove failed cannot PASS" 0 "FAIL" \
|
|
forgejo_leg_row 1 1 success 0 1
|
|
check "leg row: a runner still registered after remove cannot PASS" 0 "FAIL" \
|
|
forgejo_leg_row 1 1 success 1 0
|
|
check "leg row: no dispatch attempted, everything else clean, is PARTIAL" 0 "PARTIAL" \
|
|
forgejo_leg_row 1 1 none 1 1
|
|
check "leg row: a job that was never taken is a FAIL" 0 "FAIL" \
|
|
forgejo_leg_row 1 1 timeout 1 1
|
|
check "leg row: PARTIAL requires a clean lifecycle too" 0 "FAIL" \
|
|
forgejo_leg_row 0 1 none 1 1
|
|
|
|
# The end-to-end shape codex/grok/kimi asked for, composed the way the leg
|
|
# composes it: a tasks payload carrying a NEWER successful run (as a foreign
|
|
# drill-labeled runner would produce) must still not yield a PASS row when the
|
|
# drill's own install failed. This is the exact false-evidence case.
|
|
row_after_failed_install() {
|
|
forgejo_leg_row 0 1 "$(forgejo_run_verdict "$(forgejo_max_task_id "$1")" "$1")" 1 1
|
|
}
|
|
printf '%s' '{"workflow_runs":[{"id":24,"status":"success"},{"id":99,"status":"success"}],"total_count":2}' \
|
|
> "$FJ/foreign-runner-took-it.json"
|
|
check "install failed + a newer successful run in the payload is still FAIL, never PASS" 0 "FAIL" \
|
|
row_after_failed_install "$FJ/foreign-runner-took-it.json"
|
|
# …and the same payload with a clean lifecycle is the PASS, so the check above
|
|
# is discriminating rather than always-FAIL.
|
|
row_after_clean_install() {
|
|
forgejo_leg_row 1 1 "$(forgejo_run_verdict "" "$1")" 1 1
|
|
}
|
|
check "…while the same payload with a clean lifecycle does PASS" 0 "PASS" \
|
|
row_after_clean_install "$FJ/foreign-runner-took-it.json"
|
|
|
|
# =============================================================================
|
|
# forgejo_token_verdict — a configured leg that cannot mint must FAIL, not SKIP
|
|
# =============================================================================
|
|
# #129's own acceptance: "Token source present but the instance is unreachable
|
|
# -> the leg FAILS; it must not skip and must not pass". A mint that returns
|
|
# nothing because the instance is unreachable, the token is under-scoped or the
|
|
# repo name is wrong is a CONFIGURED leg failing — reporting "no token source"
|
|
# sends the operator to check an env var they already set, and writes SKIPPED
|
|
# where the record owes a FAIL. That is the UNREADABLE-vs-NONE shape
|
|
# drills/README.md names.
|
|
check "token: a resolved registration token is ok" 0 "ok" \
|
|
forgejo_token_verdict "reg-tok" ""
|
|
check "token: an explicit token wins even with no API token" 0 "ok" \
|
|
forgejo_token_verdict "reg-tok" ""
|
|
check "token: no token at all and no API token is a genuine SKIP" 0 "no-source" \
|
|
forgejo_token_verdict "" ""
|
|
check "token: API token offered but mint produced nothing is a FAILURE" 0 "mint-failed" \
|
|
forgejo_token_verdict "" "api-tok"
|
|
|
|
# The anti-false-positive guard, stated as its own case: an OLD completed run
|
|
# with the SAME id as pre_id must never be read as this dispatch's result.
|
|
check "verdict: a pre-existing success with the pre-id is NOT our run" 0 "pending" \
|
|
forgejo_run_verdict "25" "$FJ/new-success.json"
|
|
|
|
|
|
# Arg refusals fire before the root check (repo doctrine, bootstrap.sh:114),
|
|
# which is what makes them provable here without a throwaway machine.
|
|
check "drill.sh refuses to run without BOTH refs pinned (#103)" 2 "--box-ref" \
|
|
env -u RIG_REF -u BOX_REF bash "$ROOT/drill/drill.sh" --rig-ref release/9.9.9 --yes
|
|
check "…and the refusal shows which ref is missing" 2 "<unset>" \
|
|
env -u RIG_REF -u BOX_REF bash "$ROOT/drill/drill.sh" --rig-ref release/9.9.9 --yes
|
|
check "a tenant role is refused — the drill converges machines, not guests" 2 "not a machine role" \
|
|
bash "$ROOT/drill/drill.sh" --rig-ref r --box-ref b --role claude-box --yes
|
|
check "no --users is a refusal, naming why the drill will not default it" 2 "--users <path> is required" \
|
|
bash "$ROOT/drill/drill.sh" --rig-ref r --box-ref b --yes
|
|
check "an unreadable users file dies before anything is spent" 2 "cannot read users file" \
|
|
bash "$ROOT/drill/drill.sh" --rig-ref r --box-ref b --users "$WORK/no-such-users" --yes
|
|
check "an unknown flag dies loudly, exit 2" 2 "unknown option" \
|
|
bash "$ROOT/drill/drill.sh" --frobnicate
|
|
check "--help prints the header and exits 0" 0 "THROWAWAY" \
|
|
bash "$ROOT/drill/drill.sh" --help
|
|
check "--forgejo-instance is a known flag (the leg's opt-in)" 2 "--users <path> is required" \
|
|
bash "$ROOT/drill/drill.sh" --rig-ref r --box-ref b --forgejo-instance https://f.example.com --yes
|
|
check "--forgejo-runner-repo is a known flag" 2 "--users <path> is required" \
|
|
bash "$ROOT/drill/drill.sh" --rig-ref r --box-ref b --forgejo-runner-repo o/r --yes
|
|
check "--help names the forgejo runner leg's flags" 0 "--forgejo-instance" \
|
|
bash "$ROOT/drill/drill.sh" --help
|
|
check "--forgejo-ref is a known flag (Forgejo's dispatch needs a ref)" 2 "--users <path> is required" \
|
|
bash "$ROOT/drill/drill.sh" --rig-ref r --box-ref b --forgejo-ref dev --yes
|
|
|
|
echo "---"
|
|
echo "$PASS passed, $FAIL failed"
|
|
[ "$FAIL" -eq 0 ]
|