From 77cb4bdd64a7fba3307e410fd46f7231a4ab5a11 Mon Sep 17 00:00:00 2001 From: claude-bot-andresmgsl Date: Fri, 24 Jul 2026 00:31:53 +0000 Subject: [PATCH] test: the instrument's honesty, proven without hardware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test/drill.sh awk-extracts the harness's decision functions (the release.sh pattern) and drives them against fixtures: the ref refusal names both refs, a loud skip never classifies as a pass, the idempotence verdict is a real diff that goes non-empty when convergence is broken — demonstrated mechanically on every CI run — and the record emitter cannot produce a clean-sweep reading over a skipped leg. CI runs it in the check job. The tests caught three real harness bugs before any reviewer could: printf eating a '- '-leading format as options (a silently empty Failed section — the exact lie the record exists to prevent), tree_of trusting GNU readlink -f's exit 0 on a dangling final component, and the arg refusals sitting behind the root check in violation of the repo's own validated-before-root doctrine. (ceremony flow: issue #105) Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 5 + drill/drill.sh | 41 +++++--- test/drill.sh | 221 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 252 insertions(+), 15 deletions(-) create mode 100644 test/drill.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6734cf2..c412b37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,11 @@ jobs: run: bash test/cli.sh - name: release tests — rig's own surfaces run: bash test/release.sh + # The drill harness's honesty: refusals, the skip/pass/fail classifier, + # the idempotence capture-and-diff, the record emitter. Network-free and + # root-free — the live four-leg run is a release's drill, not CI's. + - name: drill harness tests — the instrument itself + run: bash test/drill.sh # The release guards, doctrine in heavy-duty/ceremony's README (#13's # conversion). Each one's war story — why it exists, what it refuses — # lives with its implementation upstream; the four pins below and the diff --git a/drill/drill.sh b/drill/drill.sh index 9323ff3..b460d15 100644 --- a/drill/drill.sh +++ b/drill/drill.sh @@ -121,7 +121,10 @@ run_logged() { tree_of() { local real real="$(readlink -f "$1" 2>/dev/null)" - [ -n "$real" ] || return 1 + # -e as well as -n: GNU readlink -f resolves a path whose LAST component + # does not exist (exit 0), so a dangling link would hand back a tree that + # is not there. + { [ -n "$real" ] && [ -e "$real" ]; } || return 1 dirname "$(dirname "$real")" } @@ -201,11 +204,12 @@ capture_state() { else printf ' (sshd absent)\n' fi - # Self's Tags/BackendState are the FIRST occurrences in the status JSON - # (Self serializes before Peer), and peers must not leak into the capture: - # another machine joining the tailnet between two captures is not a - # convergence diff on this one. - printf 'tailscale.self: %s\n' "$(tailscale status --json 2>/dev/null | tr -d '\n ' | grep -o '"BackendState":"[^"]*"\|"Tags":\[[^]]*\]' | head -n2 | tr '\n' ' ' || true)" + # Self's Tags is the FIRST occurrence in the status JSON (Self serializes + # before Peer). Tags only, nothing livelier: peers joining, IPs renewing + # or a backend-state flap between two captures is not a convergence diff + # on this box, and a capture that can move on its own poisons the + # idempotence verdict with noise. + printf 'tailscale.self.tags: %s\n' "$(tailscale status --json 2>/dev/null | tr -d '\n ' | grep -o '"Tags":\[[^]]*\]' | head -n1 || true)" printf 'users-ledger:\n' sed 's/^/ /' "$ledger" 2>/dev/null || printf ' (absent)\n' # Per-operator effective state: the account, its groups, its lock state, @@ -271,13 +275,16 @@ emit_record() { if [ "$fail" -eq 0 ] && [ "$skipped" -eq 0 ]; then printf '\nFailed: nothing. Every leg ran and every check passed.\n' else + # printf --: a format opening with '- ' reads as an option to bash's + # printf and emits NOTHING — a record whose Failed section silently + # vanished is exactly the lie this file exists to make impossible. [ "$fail" -gt 0 ] && printf '\nFailed:\n' for line in "${findings[@]:-}"; do - case "$line" in FAIL:*) printf '- %s\n' "$line" ;; esac + case "$line" in FAIL:*) printf -- '- %s\n' "$line" ;; esac done [ "$skipped" -gt 0 ] && printf '\nSkipped — these did NOT run, and this record is not evidence for them:\n' for line in "${findings[@]:-}"; do - case "$line" in SKIP:*) printf '- %s\n' "$line" ;; esac + case "$line" in SKIP:*) printf -- '- %s\n' "$line" ;; esac done fi printf '\nThe isolation boundary was NOT asserted here: it is box'\''s drill'\''s\n' @@ -287,11 +294,10 @@ emit_record() { # ============================================================================= # Pre-flight — every refusal this run can see coming fires here, before -# anything is installed or any credential is spent (repo doctrine: errors -# belong at the top of the run). +# anything is installed or any credential is spent. Args are validated BEFORE +# the root check (repo doctrine, bootstrap.sh:114 — so the refusals are +# testable without root, and a typo costs a re-type, never a re-ssh). # ============================================================================= -[ "$(id -u)" -eq 0 ] || { echo "drill: must run as root (bootstrap, runner, coolify and db all require it) — ssh in as root on the throwaway machine" >&2; exit 1; } - # Both refs EXPLICIT, or nothing runs. Defaulting either to main is exactly # the #103 hazard this harness exists to refuse: "I drilled the release" must # not quietly mean "I drilled whatever main was that afternoon". @@ -313,6 +319,8 @@ if [ -z "$USERS_FILE" ]; then fi [ -r "$USERS_FILE" ] || { echo "drill: cannot read users file: $USERS_FILE" >&2; exit 2; } +[ "$(id -u)" -eq 0 ] || { echo "drill: must run as root (bootstrap, runner, coolify and db all require it) — ssh in as root on the throwaway machine" >&2; exit 1; } + # The tailnet join needs a key unless this machine already joined (a re-drill # on the same throwaway). Caught here, not 10 apt-minutes into bootstrap. if [ -z "${TS_AUTHKEY:-}" ]; then @@ -416,7 +424,10 @@ if [ "$BOOTSTRAP_OK" -eq 1 ]; then uhome="$(getent passwd "$u" | cut -d: -f6)" [ -s "$uhome/.ssh/authorized_keys" ] || users_bad="$users_bad $u(no-keys)" done < <(cat "${DRILL_LEDGER:-/etc/rig/users}" 2>/dev/null) - n_users="$(grep -c ' active$' "${DRILL_LEDGER:-/etc/rig/users}" 2>/dev/null || echo 0)" + # NOT 'grep -c … || echo 0': grep -c already prints 0 on no match (and then + # exits 1), so the fallback would emit a second line into the substitution. + n_users="$(grep -c ' active$' "${DRILL_LEDGER:-/etc/rig/users}" 2>/dev/null)" || true + n_users="${n_users:-0}" [ -z "$users_bad" ] && [ "$n_users" -gt 0 ] \ && ok "operators converged: $n_users active, accounts and keys present" \ || no "operators NOT converged:${users_bad:- ledger empty}" @@ -472,11 +483,11 @@ case "$MARKER_LINE" in ok "installed box confirms: $BOXREPO@$BOXREF" if box doctor >/dev/null 2>&1; then ok "box doctor passes — setup-host converged; the host stack stands (box's own effective-state verdict)" + leg "--host yes: pinned box installed, host stack up" "PASS — $BOXREPO@$BOXREF, box doctor clean" else no "box is installed but 'box doctor' does not pass — the host stack is unproven (run 'box doctor' for box's verdict)" + leg "--host yes: pinned box installed, host stack up" "FAIL — box doctor does not pass" fi - leg "--host yes: pinned box installed, host stack up" \ - "$(box doctor >/dev/null 2>&1 && echo "PASS — $BOXREPO@$BOXREF, box doctor clean" || echo "FAIL — box doctor does not pass")" else no "no 'box' on PATH after a host=yes bootstrap — the box install did not take (bootstrap warns rather than dies there; the drill does not)" leg "--host yes: pinned box installed, host stack up" "FAIL — box CLI never landed" diff --git a/test/drill.sh b/test/drill.sh new file mode 100644 index 0000000..dbb6477 --- /dev/null +++ b/test/drill.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +# test/drill.sh — the drill harness's HONESTY, proven without hardware. +# +# drill/drill.sh is the instrument (#105), so what this suite tests is the +# instrument itself: the refusals, the classifications, the capture-and-diff +# that decides idempotence, and the record emitter — the parts whose lies +# would be believed, months later, by a reader of drills/.md. The +# four-leg live run on a real Debian machine is #107's exercise, not this +# file's: nothing here needs root, Docker, a tailnet or the network. +# +# Extraction pattern is test/release.sh's: the functions under test are +# awk-extracted from drill/drill.sh and driven against fixtures, so the tests +# exercise the shipped bytes, and the extraction check itself guards the awk +# against a drifted function boundary. +# Deliberately no `set -e` — the harness asserts on failing commands. +set -u +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" || exit 1 +PASS=0 FAIL=0 + +# check +check() { + local desc="$1" want="$2" substr="$3"; shift 3 + local out rc + out="$("$@" 2>&1)"; rc=$? + if [ "$rc" -ne "$want" ]; then + echo "FAIL: $desc — exit $rc, wanted $want" + printf '%s\n' "$out" | sed 's/^/ /' + FAIL=$((FAIL + 1)); return + fi + if [ -n "$substr" ] && ! printf '%s' "$out" | grep -qF -e "$substr"; then + echo "FAIL: $desc — output missing '$substr'" + printf '%s\n' "$out" | sed 's/^/ /' + FAIL=$((FAIL + 1)); return + fi + echo "ok: $desc"; PASS=$((PASS + 1)) +} + +# refute — the file must NOT contain the substring. +refute() { + if grep -qF -e "$2" "$3"; then + echo "FAIL: $1 — found forbidden '$2'" + FAIL=$((FAIL + 1)); return + fi + echo "ok: $1"; PASS=$((PASS + 1)) +} + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +# --- the functions under test, extracted ------------------------------------- +FNS="$WORK/drill-fns.sh" +for fn in tree_of assert_installed_from classify_leg capture_state emit_record; do + awk "/^${fn}\(\) \{/,/^\}/" "$ROOT/drill/drill.sh" >> "$FNS" +done +for fn in tree_of assert_installed_from classify_leg capture_state emit_record; do + check "extraction guards the awk: ${fn}() landed" 0 "${fn}() {" grep -F "${fn}() {" "$FNS" +done +# shellcheck source=/dev/null +. "$FNS" + +# ============================================================================= +# tree_of — the versioned tree behind a CLI's symlink chain +# ============================================================================= +IR="$WORK/install"; mkdir -p "$IR/versions/1.2.3/bin" +: > "$IR/versions/1.2.3/bin/rig" +ln -s "versions/1.2.3" "$IR/current" +mkdir -p "$WORK/bin" +ln -s "$IR/current/bin/rig" "$WORK/bin/rig" +check "tree_of resolves a current-symlink chain to versions/" 0 "$IR/versions/1.2.3" \ + tree_of "$WORK/bin/rig" +ln -s "$IR/gone/bin/rig" "$WORK/bin/dangling" +check "tree_of refuses a dangling chain — a tree that is not there is not a tree" 1 "" \ + tree_of "$WORK/bin/dangling" + +# ============================================================================= +# assert_installed_from — the up-front ref refusal, naming both refs +# ============================================================================= +TREE="$WORK/tree-main"; mkdir -p "$TREE" +printf 'heavy-duty/rig@main\n' > "$TREE/INSTALLED_FROM" +check "matching INSTALLED_FROM passes silently" 0 "" \ + assert_installed_from rig "$TREE" "heavy-duty/rig@main" +check "a mismatch refuses (the #103 hazard: asked release, got main)" 1 "FATAL" \ + assert_installed_from rig "$TREE" "heavy-duty/rig@release/9.9.9" +check "…the refusal names the ref that was ASKED for" 1 "heavy-duty/rig@release/9.9.9" \ + assert_installed_from rig "$TREE" "heavy-duty/rig@release/9.9.9" +check "…and the ref that actually LANDED" 1 "heavy-duty/rig@main" \ + assert_installed_from rig "$TREE" "heavy-duty/rig@release/9.9.9" +check "an unreadable INSTALLED_FROM refuses too — absence is not a match" 1 "" \ + assert_installed_from rig "$WORK/no-such-tree" "heavy-duty/rig@main" + +# ============================================================================= +# classify_leg — a loud skip is a SKIP, never a pass (box#153's defect class) +# ============================================================================= +printf 'skip: docker not installed — nothing to exercise\n' > "$WORK/out-skip" +printf 'ok: seeded\nok: restored\n---\n14 passed, 0 failed\n' > "$WORK/out-pass" +printf 'FAIL: restore blew up\n' > "$WORK/out-fail" +check "exit 0 + 'skip:' line classifies as skip" 0 "skip" classify_leg 0 "$WORK/out-skip" +check "exit 0, no skip line, classifies as pass" 0 "pass" classify_leg 0 "$WORK/out-pass" +check "non-zero exit classifies as fail" 0 "fail" classify_leg 1 "$WORK/out-fail" +check "a skip line cannot rescue a non-zero exit (fail wins)" 0 "fail" \ + classify_leg 1 "$WORK/out-skip" + +# ============================================================================= +# capture_state + diff — the idempotence verdict's machinery. The claim in +# #105's acceptance criteria: the assertion is a REAL diff of captured state, +# and it FAILS when convergence is broken — demonstrated here, mechanically, +# on every CI run, by breaking the state between two captures. +# ============================================================================= +FIX="$WORK/fix"; mkdir -p "$FIX/sudoers.d" +printf 'role=staging-server root-door=open host=yes join=authkey\n' > "$FIX/role" +printf 'schema=1\nbootstrapped_by=9.9.9\nbootstrapped_at=T\nconverged_by=9.9.9\nconverged_at=T\n' > "$FIX/manifest" +printf 'dan active\nghost revoked\n' > "$FIX/ledger" +printf 'APT::Periodic::Update-Package-Lists "1";\n' > "$FIX/autoup" +printf '127.0.0.1 localhost\n127.0.1.1\tstaging-server\n' > "$FIX/hosts" +printf 'nosuchdrilluser ALL=(ALL) NOPASSWD:ALL\n' > "$FIX/sudoers.d/00-rig-nosuch" +# A stubbed sshd, so the effective-config section is exercised rather than +# skipped on a box with no daemon (repo precedent: test/release.sh's curl). +STUB="$WORK/stub"; mkdir -p "$STUB" +# The single-quoted $SSHD_FIXTURE is the STUB's expansion, not this shell's. +# shellcheck disable=SC2016 +printf '#!/usr/bin/env bash\ncat "$SSHD_FIXTURE"\n' > "$STUB/sshd"; chmod +x "$STUB/sshd" +printf 'passwordauthentication no\npermitrootlogin prohibit-password\n' > "$FIX/sshd-T" + +cap() { # cap — capture_state against the fixture set + RIG_ROLE_MARKER="$FIX/role" RIG_MANIFEST="$FIX/manifest" \ + DRILL_LEDGER="$FIX/ledger" DRILL_AUTOUPGRADES="$FIX/autoup" \ + DRILL_ETC_HOSTS="$FIX/hosts" DRILL_SUDOERS_DIR="$FIX/sudoers.d" \ + SSHD_FIXTURE="$FIX/sshd-T" PATH="$STUB:$PATH" \ + bash -c '. "$1"; capture_state "$2"' _ "$FNS" "$2" 2>/dev/null + : +} +# cap runs capture_state in a child bash so the PATH stub cannot leak into +# this harness; $2 arrives as the capture's outfile. +cap out "$WORK/cap1" +cap out "$WORK/cap2" +check "two captures over untouched state diff EMPTY (the converged verdict)" 0 "" \ + diff -u "$WORK/cap1" "$WORK/cap2" +check "the capture reads the fixtures, not the machine (marker line present)" 0 "role=staging-server" \ + grep -o 'role=staging-server[^"]*' "$WORK/cap1" +check "…the sshd section captured the effective config" 0 "passwordauthentication no" \ + cat "$WORK/cap1" +check "…a ledger user with no account reads as one, deterministically" 0 "(no account)" \ + cat "$WORK/cap1" + +# Break convergence: the re-run "changed" the role marker and root's door. +printf 'role=staging-server root-door=closed host=yes join=authkey\n' > "$FIX/role" +printf 'passwordauthentication yes\npermitrootlogin prohibit-password\n' > "$FIX/sshd-T" +cap out "$WORK/cap3" +check "a broken convergence makes the diff NON-empty — the assertion can fail" 1 "root-door=closed" \ + diff -u "$WORK/cap1" "$WORK/cap3" +check "…and the diff names the drifted sshd keyword, not just 'differs'" 1 "passwordauthentication yes" \ + diff -u "$WORK/cap1" "$WORK/cap3" + +# ============================================================================= +# emit_record — the record is drills/README.md's shape, and it cannot lie: +# a failed run still emits, a skipped leg is named, no clean-sweep reading. +# ============================================================================= +emit() { # emit — emit_record with the harness globals staged + DRILL_VERSION="9.9.9" RUN_ID="drill-2026-01-01-a" \ + REF="release/9.9.9" BOXREF="release/0.4.0" RIG_SHA="5d6e7f8" BOX_SHA="1a2b3c4" \ + bash -c ' + . "$1" + pass=12 fail=1 skipped=1 + findings=("FAIL: coolify container state: absent" "SKIP: runner lifecycle: no --runner-repo fork given — the leg did not run" "NOTE: something worth a line") + LEG_NAMES=("convergence — bootstrap staging-server reaches its role" "re-converge (idempotence)" "coolify install (4.1.2)" "runner lifecycle") + LEG_RESULTS=("PASS (312s)" "clean, no changes" "FAIL — container absent" "SKIPPED — no fork provided") + emit_record "$2" + ' _ "$FNS" "$2" +} +emit out "$WORK/record.md" +check "record: the version-and-date heading" 0 "# Release drill — 9.9.9 — " head -1 "$WORK/record.md" +check "record: the run ID that joins the family's records" 0 "Run ID: drill-2026-01-01-a" cat "$WORK/record.md" +check "record: both pinned refs with their SHAs" 0 "rig@5d6e7f8 (RIG_REF=release/9.9.9)" cat "$WORK/record.md" +check "record: …box's too" 0 "box@1a2b3c4 (BOX_REF=release/0.4.0)" cat "$WORK/record.md" +check "record: one table row per leg, result verbatim" 0 "| re-converge (idempotence) | clean, no changes |" cat "$WORK/record.md" +check "record: the numbers, skips counted apart from passes" 0 "12 passed, 1 failed, 1 skipped" cat "$WORK/record.md" +check "record: a FAILED run still names what failed (evidence, not success)" 0 "FAIL: coolify container state: absent" cat "$WORK/record.md" +check "record: a skipped leg is stated as NOT run, by name" 0 "SKIP: runner lifecycle" cat "$WORK/record.md" +check "record: the skip section says the record is not evidence for it" 0 "not evidence" cat "$WORK/record.md" +check "record: the isolation boundary is named as box's, in words" 0 "NOT asserted here" cat "$WORK/record.md" +refute "record with a skip cannot read as a clean sweep" "Failed: nothing" "$WORK/record.md" +refute "notes are findings for the log, not failures for the record" "NOTE: something" "$WORK/record.md" + +# The all-green shape: says so plainly, and only then. +DRILL_VERSION="9.9.9" RUN_ID="drill-2026-01-01-a" \ +REF="release/9.9.9" BOXREF="release/0.4.0" RIG_SHA="5d6e7f8" BOX_SHA="1a2b3c4" \ +bash -c ' + . "$1" + pass=20 fail=0 skipped=0 + findings=() + LEG_NAMES=("convergence" "re-converge (idempotence)") + LEG_RESULTS=("PASS" "clean, no changes") + emit_record "$2" +' _ "$FNS" "$WORK/record-green.md" +check "an all-green record says every leg ran and passed" 0 "Every leg ran and every check passed" \ + cat "$WORK/record-green.md" + +# ============================================================================= +# the shipped script itself +# ============================================================================= +# Arg refusals fire before the root check (repo doctrine, bootstrap.sh:114), +# which is what makes them provable here without a throwaway machine. +check "drill.sh refuses to run without BOTH refs pinned (#103)" 2 "--box-ref" \ + env -u RIG_REF -u BOX_REF bash "$ROOT/drill/drill.sh" --rig-ref release/9.9.9 --yes +check "…and the refusal shows which ref is missing" 2 "" \ + env -u RIG_REF -u BOX_REF bash "$ROOT/drill/drill.sh" --rig-ref release/9.9.9 --yes +check "a tenant role is refused — the drill converges machines, not guests" 2 "not a machine role" \ + bash "$ROOT/drill/drill.sh" --rig-ref r --box-ref b --role claude-box --yes +check "no --users is a refusal, naming why the drill will not default it" 2 "--users is required" \ + bash "$ROOT/drill/drill.sh" --rig-ref r --box-ref b --yes +check "an unreadable users file dies before anything is spent" 2 "cannot read users file" \ + bash "$ROOT/drill/drill.sh" --rig-ref r --box-ref b --users "$WORK/no-such-users" --yes +check "an unknown flag dies loudly, exit 2" 2 "unknown option" \ + bash "$ROOT/drill/drill.sh" --frobnicate +check "--help prints the header and exits 0" 0 "THROWAWAY" \ + bash "$ROOT/drill/drill.sh" --help + +echo "---" +echo "$PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ]