From 6241e87538a9764ff3e0cc4e8dca84ed73a36312 Mon Sep 17 00:00:00 2001 From: claude-bot-andresmgsl Date: Fri, 24 Jul 2026 00:26:58 +0000 Subject: [PATCH 1/4] =?UTF-8?q?feat:=20drill/drill.sh=20=E2=80=94=20the=20?= =?UTF-8?q?instrument=20rig's=20drill=20gate=20never=20had?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Box's shape, rig's legs: the reporting verbs and set -u-only discipline (a failing check is data, not a crash), a fatal INSTALLED_FROM assertion on BOTH pinned refs before anything is believed, convergence asserted on effective state, idempotence decided by a mechanical capture-and-diff, db driven through test/db-integration.sh with its loud-skip contract kept, the runner lifecycle against a fork, a pinned coolify install, and a record emitter that writes drills/.md in the schema the drill-recorded gate reads — skips counted and named, never folded into passes. The --host yes leg stops at 'the pinned box installed and its host stack stands', in as many words in the output: the isolation boundary is box's drill's assertion, joined to this record by the shared run ID. (ceremony flow: issue #105) Co-Authored-By: Claude Fable 5 --- drill/drill.sh | 656 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 656 insertions(+) create mode 100644 drill/drill.sh diff --git a/drill/drill.sh b/drill/drill.sh new file mode 100644 index 0000000..9323ff3 --- /dev/null +++ b/drill/drill.sh @@ -0,0 +1,656 @@ +#!/usr/bin/env bash +# drill/drill.sh — rig's release drill: the instrument behind drills/README.md. +# +# ⚠ DESTRUCTIVE, AND MEANT TO BE. Run it on a THROWAWAY Debian machine you +# can format. It wipes any installed rig and reinstalls from the pinned +# ref, hardens sshd, sets the hostname, joins the tailnet, installs box +# and its Incus stack, installs Coolify and a GitHub Actions runner. +# Never run it on a machine you care about. +# +# TS_AUTHKEY=tskey-... bash drill/drill.sh \ +# --rig-ref release/0.4.0 --box-ref release/0.10.0 \ +# --users ./drill-users --run-id drill-2026-07-24-a \ +# --coolify-version 4.1.2 --runner-repo you/rig --yes +# +# rig's drill asserts CONVERGENCE — a machine reaches its role, idempotently. +# The legs (drills/README.md, issue #105): +# +# 1. convergence + idempotence — `rig bootstrap --users ` +# reaches the declared role; a re-run produces an EMPTY state diff, +# mechanically, never by eye. Rides along: the --host yes assertions +# (the pinned box installed, its host stack stands — and it STOPS there; +# the isolation boundary is box's drill's assertion, not this one's). +# 2. db — the real dump/restore round-trip, test/db-integration.sh. +# 3. runner lifecycle — register, take a job, deregister, against a fork. +# 4. coolify install — at a pinned version, AUTOUPDATE=false. +# +# Execution order is 1, 4, 2, 3 — coolify's installer is what puts Docker on +# the box, and leg 2 needs a daemon; running db before coolify would skip a +# leg this same run makes runnable. The record lists legs as they ran. +# +# Exit 0 = no check failed. A FAILED drill still emits a complete record — +# the gate wants evidence, not success — and skipped legs are counted and +# named, never folded into the passes (heavy-duty/box#153's defect class). +# +# The file is one long 'probe && ok "…" || no "…"'. ok/no always return 0, so +# the C-may-run-when-A-is-true trap SC2015 warns about cannot fire here. +# shellcheck disable=SC2015 +# +# NOT -e: a failing check is data, not a crash — a drill that aborts on its +# first failure reports one problem per afternoon. NOT pipefail: checks of the +# 'refusal 2>&1 | grep -q text' shape have a left side that exits non-zero BY +# DESIGN, and 'grep -q' SIGPIPEs the left side on early match — box's first +# live run turned both into false FAILs. The pipeline verdict must be grep's +# alone. (box drill/drill.sh's header, the discipline #105 prescribes.) +set -u + +SELF="$(readlink -f "$0")" +ROOT="$(cd "$(dirname "$SELF")/.." && pwd)" + +REPO="${RIG_REPO:-heavy-duty/rig}" +REF="${RIG_REF:-}" +BOXREPO="${BOX_REPO:-heavy-duty/box}" +BOXREF="${BOX_REF:-}" +ROLE=staging-server +USERS_FILE="${DRILL_USERS_FILE:-}" +RUN_ID="${DRILL_RUN_ID:-drill-$(date -u +%F)}" +RECORD="${DRILL_RECORD:-}" +COOLIFY_VERSION="${DRILL_COOLIFY_VERSION:-}" +RUNNER_REPO="${DRILL_RUNNER_REPO:-}" +RUNNER_WORKFLOW="${DRILL_RUNNER_WORKFLOW:-drill.yml}" +YES=0 + +while [ $# -gt 0 ]; do + case "$1" in + --yes|-y) YES=1; shift ;; + --rig-repo) REPO="$2"; shift 2 ;; + --rig-ref) REF="$2"; shift 2 ;; + --box-repo) BOXREPO="$2"; shift 2 ;; + --box-ref) BOXREF="$2"; shift 2 ;; + --role) ROLE="$2"; shift 2 ;; + --users) USERS_FILE="$2"; shift 2 ;; + --run-id) RUN_ID="$2"; shift 2 ;; + --record) RECORD="$2"; shift 2 ;; + --coolify-version) COOLIFY_VERSION="$2"; shift 2 ;; + --runner-repo) RUNNER_REPO="$2"; shift 2 ;; + --runner-workflow) RUNNER_WORKFLOW="$2"; shift 2 ;; + -h|--help) sed -n '2,36p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "drill: unknown option: $1 (see --help)" >&2; exit 2 ;; + esac +done + +# --- the reporting verbs (box drill/drill.sh:52-58, the parts worth copying) -- +# ok/no/skip/note always return 0: the body stays one long sequence of +# 'probe && ok || no' without fighting the shell. SKIP is its own verb and its +# own counter — a leg that did not run must be visually and arithmetically +# distinct from one that passed (box#153's defect class: a silent skip reads +# as a pass in the record, months later). +pass=0; fail=0; skipped=0; findings=() +ok() { printf ' \033[32mPASS\033[0m %s\n' "$*"; pass=$((pass + 1)); } +no() { printf ' \033[31mFAIL\033[0m %s\n' "$*"; fail=$((fail + 1)); findings+=("FAIL: $*"); } +skip() { printf ' \033[35mSKIP\033[0m %s\n' "$*"; skipped=$((skipped + 1)); findings+=("SKIP: $*"); } +note() { printf ' \033[33mNOTE\033[0m %s\n' "$*"; findings+=("NOTE: $*"); } +inf() { printf ' %s\n' "$*"; } +phase(){ printf '\n\033[1m══ %s\033[0m\n' "$*"; } + +# The record's leg table, appended as legs run. One row per leg, result text +# written at the moment the leg's verdict is known — never reconstructed from +# memory at the end (an invented number is worse than no number). +LEG_NAMES=(); LEG_RESULTS=() +leg() { LEG_NAMES+=("$1"); LEG_RESULTS+=("$2"); } + +# run_logged — run a long command with its narration in a file +# and a dot every 5s on the terminal: a silent multi-minute apt/install run is +# indistinguishable from a wedge, and that ambiguity has cost box whole +# evenings. Returns the command's exit code. +run_logged() { + local log="$1"; shift + inf "watch it live in another terminal: tail -f $log" + "$@" >"$log" 2>&1 /dev/null; do printf '.'; sleep 5; done + printf '\n' + wait "$pid" +} + +# tree_of — the versioned install tree a CLI's symlink chain lands +# in. Both rig and box install as /versions//bin/ behind a +# 'current' link, so the tree is two dirnames above the resolved binary — +# derived from the chain itself, never from a hardcoded install root (root vs +# user installs put the root in different places). +tree_of() { + local real + real="$(readlink -f "$1" 2>/dev/null)" + [ -n "$real" ] || return 1 + dirname "$(dirname "$real")" +} + +# assert_installed_from — ASSERT WHAT LANDED, never trust +# that the install obeyed. An installer invoked with stale env vars silently +# falls back to its defaults (rig#103: both BOX_REF and RIG_REF default to +# main), and a drill that thinks it exercised release/X but actually got main +# has proven nothing about the combination that ships — worse than one that +# fails, because the record it leaves LOOKS like evidence. Refusal names both +# refs, per #105's acceptance criteria. +assert_installed_from() { + local what="$1" tree="$2" want="$3" got + got="$(cat "$tree/INSTALLED_FROM" 2>/dev/null || echo '')" + if [ "$got" != "$want" ]; then + printf 'drill: FATAL — asked to install %s from %s, but the installed tree says %s.\n' "$what" "$want" "$got" >&2 + printf ' (tree: %s)\n' "$tree" >&2 + printf ' A drill that silently drills the wrong code is worse than one that fails:\n' >&2 + printf ' every result below would describe a tree that is not the candidate. Check\n' >&2 + printf ' the env this drill inherited (a stale RIG_REF/BOX_REF export), fix the\n' >&2 + printf ' pin, and re-run.\n' >&2 + return 1 + fi + return 0 +} + +# classify_leg — pass | skip | fail. The skip contract is +# test/db-integration.sh's, copied carefully: it skips CLEANLY (exit 0) with a +# 'skip: ' line when it cannot run, so exit code alone reads a +# not-run leg as a pass. The reason line is the verdict's tiebreaker; a +# non-zero exit is a fail whatever the output says (a die after a skip line +# would be a broken harness, not a skip). +classify_leg() { + local rc="$1" out="$2" + if [ "$rc" -eq 0 ] && grep -q '^skip:' "$out" 2>/dev/null; then + printf 'skip' + elif [ "$rc" -eq 0 ]; then + printf 'pass' + else + printf 'fail' + fi +} + +# capture_state — the convergent surface bootstrap owns, as one +# diffable text file. Leg 1's idempotence claim is decided by capturing this +# BEFORE and AFTER the re-run and diffing — mechanically, because idempotence +# is the single easiest property to convince yourself of by eye (#105). +# +# What is captured is what bootstrap CONVERGES, nothing that legitimately +# moves between two back-to-back runs: no package lists (unattended-upgrades +# may act between captures), no clocks. The manifest is included WHOLE on +# purpose — lib/manifest.sh's contract is that a same-version re-run renders +# byte-identical content (converged_at tracks the version, not the run), so +# the diff ENFORCES that contract instead of exempting it. +# +# Every path is overridable so test/drill.sh proves the capture-and-diff +# machinery against fixtures, without root (repo precedent: RIG_ROLE_MARKER, +# RIG_MANIFEST). Absent files and commands degrade to a deterministic +# '(absent)' — a capture must never fail, only describe. +capture_state() { + local out="$1" marker manifest ledger autoup hosts u state home keys + marker="${RIG_ROLE_MARKER:-/etc/rig/role}" + manifest="${RIG_MANIFEST:-/etc/rig/manifest}" + ledger="${DRILL_LEDGER:-/etc/rig/users}" + autoup="${DRILL_AUTOUPGRADES:-/etc/apt/apt.conf.d/20auto-upgrades}" + hosts="${DRILL_ETC_HOSTS:-/etc/hosts}" + { + printf 'hostname: %s\n' "$(hostname 2>/dev/null || echo '(absent)')" + printf 'hosts.127.0.1.1: %s\n' "$(grep -E '^127\.0\.1\.1[[:space:]]' "$hosts" 2>/dev/null || echo '(absent)')" + printf 'role-marker: %s\n' "$(cat "$marker" 2>/dev/null || echo '(absent)')" + printf 'manifest:\n' + sed 's/^/ /' "$manifest" 2>/dev/null || printf ' (absent)\n' + printf 'auto-upgrades:\n' + sed 's/^/ /' "$autoup" 2>/dev/null || printf ' (absent)\n' + printf 'sshd-effective:\n' + if command -v sshd >/dev/null 2>&1; then + sshd -T 2>/dev/null | sort | sed 's/^/ /' || printf ' (sshd -T failed)\n' + else + printf ' (sshd absent)\n' + fi + # Self's Tags/BackendState are the FIRST occurrences in the status JSON + # (Self serializes before Peer), and peers must not leak into the capture: + # another machine joining the tailnet between two captures is not a + # convergence diff on this one. + printf 'tailscale.self: %s\n' "$(tailscale status --json 2>/dev/null | tr -d '\n ' | grep -o '"BackendState":"[^"]*"\|"Tags":\[[^]]*\]' | head -n2 | tr '\n' ' ' || true)" + printf 'users-ledger:\n' + sed 's/^/ /' "$ledger" 2>/dev/null || printf ' (absent)\n' + # Per-operator effective state: the account, its groups, its lock state, + # its keys. sha256 of authorized_keys, not the keys themselves — the + # capture may end up quoted in a record and keys are long, not secret. + while read -r u state; do + [ -n "${u:-}" ] || continue + if ! id -u "$u" >/dev/null 2>&1; then + printf 'user.%s: (no account)\n' "$u" + continue + fi + printf 'user.%s: state=%s groups=%s lock=%s\n' "$u" "${state:-active}" \ + "$(id -Gn "$u" 2>/dev/null | tr ' ' ',')" \ + "$(passwd -S "$u" 2>/dev/null | awk '{print $2}' || echo '?')" + home="$(getent passwd "$u" | cut -d: -f6)" + keys="$home/.ssh/authorized_keys" + printf 'user.%s.authorized_keys: %s\n' "$u" \ + "$(sha256sum "$keys" 2>/dev/null | cut -d' ' -f1 || echo '(none)')" + done < <(cat "$ledger" 2>/dev/null) + printf 'sudoers.d:\n' + find "${DRILL_SUDOERS_DIR:-/etc/sudoers.d}" -maxdepth 1 -type f 2>/dev/null | sort \ + | while read -r u; do printf ' %s %s\n' "$(sha256sum "$u" | cut -d' ' -f1)" "$u"; done + printf 'box: %s\n' "$(command -v box 2>/dev/null || echo '(absent)')" + } > "$out" +} + +# ref_sha — the commit the record cites. Tags outrank +# branches (the installer's own precedence, install.sh:117-120). Resolved once +# up front and reused, so the record and the install describe the same instant +# even if the branch moves mid-drill. Empty on failure; the record then says +# 'unresolved' rather than inventing one. +ref_sha() { + local sha + sha="$(git ls-remote "https://github.com/$1" "refs/tags/$2" 2>/dev/null | head -n1 | cut -f1)" + [ -n "$sha" ] || sha="$(git ls-remote "https://github.com/$1" "refs/heads/$2" 2>/dev/null | head -n1 | cut -f1)" + printf '%s' "${sha:0:7}" +} + +# emit_record — drills/.md, in the shape drills/README.md +# defines: what ran, on what host, the pinned refs and SHAs, the numbers, and +# what failed. Emitted on EVERY completed run — a failed drill is still a +# valid record; the gate wants evidence, not success. Skipped legs are listed +# by name: a record with no failures listed reads as "nothing broke", so a leg +# that was not run says so instead of being omitted. +emit_record() { + local out="$1" i os cpus ram virt line + os="$(. /etc/os-release 2>/dev/null && printf '%s' "${PRETTY_NAME:-unknown}")" + cpus="$(nproc 2>/dev/null || echo '?')" + ram="$(awk '/MemTotal/{printf "%.0f", $2/1024/1024}' /proc/meminfo 2>/dev/null || echo '?')" + virt="$(systemd-detect-virt 2>/dev/null || echo unknown)" + { + printf '# Release drill — %s — %s\n\n' "$DRILL_VERSION" "$(date -u +%F)" + printf 'Run ID: %s. Host: %s, %s vCPU / %s GB RAM (%s).\n' "$RUN_ID" "${os:-unknown}" "$cpus" "$ram" "$virt" + printf 'Candidate refs: rig@%s (RIG_REF=%s), box@%s (BOX_REF=%s).\n' \ + "${RIG_SHA:-unresolved}" "$REF" "${BOX_SHA:-unresolved}" "$BOXREF" + printf 'Instrument: drill/drill.sh, legs in execution order.\n\n' + printf '| Leg | Result |\n' + printf '| --- | --- |\n' + for i in "${!LEG_NAMES[@]}"; do + printf '| %s | %s |\n' "${LEG_NAMES[$i]}" "${LEG_RESULTS[$i]}" + done + printf '\nChecks: %s passed, %s failed, %s skipped.\n' "$pass" "$fail" "$skipped" + if [ "$fail" -eq 0 ] && [ "$skipped" -eq 0 ]; then + printf '\nFailed: nothing. Every leg ran and every check passed.\n' + else + [ "$fail" -gt 0 ] && printf '\nFailed:\n' + for line in "${findings[@]:-}"; do + case "$line" in FAIL:*) printf '- %s\n' "$line" ;; esac + done + [ "$skipped" -gt 0 ] && printf '\nSkipped — these did NOT run, and this record is not evidence for them:\n' + for line in "${findings[@]:-}"; do + case "$line" in SKIP:*) printf '- %s\n' "$line" ;; esac + done + fi + printf '\nThe isolation boundary was NOT asserted here: it is box'\''s drill'\''s\n' + printf 'assertion (heavy-duty/box drill/drill.sh), joined to this record by the run ID.\n' + } > "$out" +} + +# ============================================================================= +# Pre-flight — every refusal this run can see coming fires here, before +# anything is installed or any credential is spent (repo doctrine: errors +# belong at the top of the run). +# ============================================================================= +[ "$(id -u)" -eq 0 ] || { echo "drill: must run as root (bootstrap, runner, coolify and db all require it) — ssh in as root on the throwaway machine" >&2; exit 1; } + +# Both refs EXPLICIT, or nothing runs. Defaulting either to main is exactly +# the #103 hazard this harness exists to refuse: "I drilled the release" must +# not quietly mean "I drilled whatever main was that afternoon". +if [ -z "$REF" ] || [ -z "$BOXREF" ]; then + echo "drill: both refs must be pinned explicitly — a drill against an unstated ref is not evidence (#103):" >&2 + echo " --rig-ref (or RIG_REF) the rig candidate, e.g. release/0.4.0 [got: ${REF:-}]" >&2 + echo " --box-ref (or BOX_REF) the box that will ship with it [got: ${BOXREF:-}]" >&2 + exit 2 +fi + +case "$ROLE" in + staging-server|dev-server|control-plane-server|workload-server|runner-server) ;; + *) echo "drill: --role $ROLE is not a machine role this drill can converge unattended" >&2; exit 2 ;; +esac + +if [ -z "$USERS_FILE" ]; then + echo "drill: --users is required — leg 1 asserts operators converged, and bootstrap requires the file (its --no-users opt-out would leave leg 1 asserting nothing)" >&2 + exit 2 +fi +[ -r "$USERS_FILE" ] || { echo "drill: cannot read users file: $USERS_FILE" >&2; exit 2; } + +# The tailnet join needs a key unless this machine already joined (a re-drill +# on the same throwaway). Caught here, not 10 apt-minutes into bootstrap. +if [ -z "${TS_AUTHKEY:-}" ]; then + if ! { command -v tailscale >/dev/null 2>&1 && tailscale status >/dev/null 2>&1; }; then + echo "drill: TS_AUTHKEY is unset and this machine has not joined a tailnet — leg 1's bootstrap will refuse. Mint a single-use TAGGED pre-auth key and export TS_AUTHKEY." >&2 + exit 2 + fi +fi + +command -v curl >/dev/null 2>&1 || { echo "drill: curl is required (the pinned installs download over it)" >&2; exit 1; } + +if [ "$YES" -ne 1 ]; then + cat <&2; exit 2; } + printf 'Continue? [y/N] ' + read -r reply + case "$reply" in y|Y|yes) ;; *) echo "stopped."; exit 1 ;; esac +fi + +phase "Pinned candidates" +RIG_SHA="$(ref_sha "$REPO" "$REF")" +BOX_SHA="$(ref_sha "$BOXREPO" "$BOXREF")" +inf "rig: $REPO@$REF (${RIG_SHA:-unresolved})" +inf "box: $BOXREPO@$BOXREF (${BOX_SHA:-unresolved})" +inf "run ID: $RUN_ID — drills sharing this substrate share it (drills/README.md)" + +# ============================================================================= +phase "Installing rig ($REPO@$REF) from scratch" +# ============================================================================= +# The drill proves a tree from SCRATCH every run — a fresh machine, not a +# converged install — so any prior rig goes first (root's install lands at +# \$HOME/.local/share/rig with the /usr/local/bin symlink). +rm -rf "$HOME/.local/share/rig" /usr/local/bin/rig + +if ! run_logged /tmp/drill-rig-install.log \ + env RIG_REPO="$REPO" RIG_REF="$REF" \ + bash -c "bash <(curl -fsSL \"https://raw.githubusercontent.com/$REPO/$REF/install.sh\")"; then + echo "drill: rig's installer failed — tail of /tmp/drill-rig-install.log:" >&2 + tail -5 /tmp/drill-rig-install.log >&2 + exit 1 +fi +command -v rig >/dev/null 2>&1 || { echo "drill: installer reported success but no 'rig' on PATH" >&2; exit 1; } + +# ASSERT WHAT LANDED — the up-front ref assertion, fatal on mismatch. +RIG_TREE="$(tree_of "$(command -v rig)")" +assert_installed_from rig "$RIG_TREE" "$REPO@$REF" || exit 1 +DRILL_VERSION="$(head -n1 "$RIG_TREE/VERSION" 2>/dev/null || echo unknown)" +ok "installed tree confirms: $REPO@$REF (version $DRILL_VERSION)" +[ -n "$RECORD" ] || RECORD="$ROOT/drills/$DRILL_VERSION.md" + +# ============================================================================= +phase "Leg 1 — convergence: rig bootstrap $ROLE" +# ============================================================================= +# BOX_REPO/BOX_REF ride the environment into bootstrap's host=yes box install, +# so the box that lands is the pinned candidate, not box's default (main). +export BOX_REPO="$BOXREPO" BOX_REF="$BOXREF" + +t0=$SECONDS +if run_logged /tmp/drill-bootstrap-1.log rig bootstrap "$ROLE" --users "$USERS_FILE"; then + ok "rig bootstrap $ROLE --users … exited 0 ($((SECONDS - t0))s)" + BOOTSTRAP_OK=1 +else + no "rig bootstrap $ROLE FAILED — tail: $(tail -3 /tmp/drill-bootstrap-1.log | tr '\n' ' ')" + BOOTSTRAP_OK=0 +fi + +MARKER_LINE="$(cat "${RIG_ROLE_MARKER:-/etc/rig/role}" 2>/dev/null || true)" +if [ "$BOOTSTRAP_OK" -eq 1 ]; then + # The role, asserted on EFFECTIVE state — the marker, the daemon's resolved + # config, the netmap's granted tags — never on what was requested (the + # sshd-first-wins lesson, lib/sshd.sh:63-70). + case "$MARKER_LINE" in + "role=$ROLE "*) ok "role marker: $MARKER_LINE" ;; + *) no "role marker is '$MARKER_LINE' — expected role=$ROLE …" ;; + esac + sshd -T 2>/dev/null | grep -qx 'passwordauthentication no' \ + && ok "sshd -T resolves passwordauthentication no (the hardening took)" \ + || no "sshd still resolves password auth — the 00-rig.conf drop-in is not winning" + ts_tags="$(tailscale status --json 2>/dev/null | tr -d '\n ' | grep -o '"Tags":\[[^]]*\]' | head -n1)" + if [ -n "$ts_tags" ] && [ "$ts_tags" != '"Tags":[]' ]; then + ok "tailnet joined, tagged: $ts_tags" + else + no "tailnet join did not leave a tagged node (got: ${ts_tags:-nothing}) — bootstrap's verify should have refused this" + fi + grep -q 'Unattended-Upgrade "1"' /etc/apt/apt.conf.d/20auto-upgrades 2>/dev/null \ + && ok "unattended-upgrades enabled" || no "20auto-upgrades missing or wrong" + grep -q "converged_by=$DRILL_VERSION" "${RIG_MANIFEST:-/etc/rig/manifest}" 2>/dev/null \ + && ok "manifest: converged_by=$DRILL_VERSION" || no "manifest does not name $DRILL_VERSION as the converging rig" + users_bad="" + while read -r u state; do + [ "$state" = active ] || continue + id -u "$u" >/dev/null 2>&1 || { users_bad="$users_bad $u(no-account)"; continue; } + uhome="$(getent passwd "$u" | cut -d: -f6)" + [ -s "$uhome/.ssh/authorized_keys" ] || users_bad="$users_bad $u(no-keys)" + done < <(cat "${DRILL_LEDGER:-/etc/rig/users}" 2>/dev/null) + n_users="$(grep -c ' active$' "${DRILL_LEDGER:-/etc/rig/users}" 2>/dev/null || echo 0)" + [ -z "$users_bad" ] && [ "$n_users" -gt 0 ] \ + && ok "operators converged: $n_users active, accounts and keys present" \ + || no "operators NOT converged:${users_bad:- ledger empty}" + leg "convergence — bootstrap $ROLE reaches its role" \ + "$([ "$fail" -eq 0 ] && echo "PASS ($((SECONDS - t0))s)" || echo "FAIL — see Failed below")" + + # --- idempotence: the claim this drill exists to make ---------------------- + # Capture, re-run, capture, diff. Mechanically — never "watched it not + # obviously break". An empty diff IS the definition of converged. + phase "Leg 1 — idempotence: the re-run must change nothing" + pre="$(mktemp)"; post="$(mktemp)" + capture_state "$pre" + t0=$SECONDS + if run_logged /tmp/drill-bootstrap-2.log rig bootstrap "$ROLE" --users "$USERS_FILE"; then + ok "second bootstrap exited 0 ($((SECONDS - t0))s)" + else + no "second bootstrap FAILED — tail: $(tail -3 /tmp/drill-bootstrap-2.log | tr '\n' ' ')" + fi + capture_state "$post" + if statediff="$(diff -u "$pre" "$post")"; then + ok "re-converge is a no-op: the state diff is empty" + leg "re-converge (idempotence)" "clean, no changes" + else + dlines="$(printf '%s\n' "$statediff" | grep -c '^[+-][^+-]')" + no "re-converge CHANGED the box — $dlines state line(s) differ:" + printf '%s\n' "$statediff" | sed 's/^/ /' + leg "re-converge (idempotence)" "DIRTY — $dlines state line(s) changed on the re-run" + fi + rm -f "$pre" "$post" +else + leg "convergence — bootstrap $ROLE reaches its role" "FAIL — bootstrap exited non-zero" + skip "idempotence not asserted — the first converge already failed, a re-run diff would measure noise" + leg "re-converge (idempotence)" "SKIPPED — first converge failed" +fi + +# ============================================================================= +phase "--host yes — the box that will ship" +# ============================================================================= +# The assertions #105 settles this leg at: the installer ran, INSTALLED_FROM +# matches the requested BOX_REF, setup-host exited clean, the stack it claims +# stands. Then it STOPS. Not one isolation probe: two records that both claim +# the trust boundary will eventually disagree with no tiebreaker, and a +# partial isolation check reads — months later, in a record — as though the +# boundary was drilled (box#153's shape through a different door). Resist +# adding "just one" probe here; that is box's drill's whole job. +case "$MARKER_LINE" in + *"host=yes"*) + if command -v box >/dev/null 2>&1; then + ok "box CLI on PATH" + BOX_TREE="$(tree_of "$(command -v box)")" + # Fatal, like rig's own: a wrong box under --host yes poisons the pair. + assert_installed_from box "$BOX_TREE" "$BOXREPO@$BOXREF" || exit 1 + ok "installed box confirms: $BOXREPO@$BOXREF" + if box doctor >/dev/null 2>&1; then + ok "box doctor passes — setup-host converged; the host stack stands (box's own effective-state verdict)" + else + no "box is installed but 'box doctor' does not pass — the host stack is unproven (run 'box doctor' for box's verdict)" + fi + leg "--host yes: pinned box installed, host stack up" \ + "$(box doctor >/dev/null 2>&1 && echo "PASS — $BOXREPO@$BOXREF, box doctor clean" || echo "FAIL — box doctor does not pass")" + else + no "no 'box' on PATH after a host=yes bootstrap — the box install did not take (bootstrap warns rather than dies there; the drill does not)" + leg "--host yes: pinned box installed, host stack up" "FAIL — box CLI never landed" + fi + inf "isolation NOT asserted here — deliberately. The VM trust boundary is box's" + inf "assertion, made by box's own drill (~85 probes); this leg stops at 'the pinned" + inf "box installed and its host stack stands'. The records join on the run ID." + ;; + *) + skip "--host yes assertions: role $ROLE left host=no (marker: ${MARKER_LINE:-absent})" + leg "--host yes: pinned box installed, host stack up" "SKIPPED — this role does not host VMs" + ;; +esac + +# ============================================================================= +phase "Leg 4 — coolify install (pinned, AUTOUPDATE=false)" +# ============================================================================= +# Runs BEFORE leg 2 on purpose: Coolify's installer is what puts Docker on the +# box, and the db leg needs a daemon — ordering them the other way around +# would manufacture a skip this same run could have avoided. +if [ -z "$COOLIFY_VERSION" ]; then + skip "coolify install: no --coolify-version pin given — the leg did not run (rig's own install refuses to default a version, and so does its drill)" + leg "coolify install" "SKIPPED — no version pin provided" +else + t0=$SECONDS + if run_logged /tmp/drill-coolify.log rig coolify install --version "$COOLIFY_VERSION"; then + ok "rig coolify install --version $COOLIFY_VERSION exited 0 ($((SECONDS - t0))s)" + grep -qx 'AUTOUPDATE=false' /data/coolify/source/.env 2>/dev/null \ + && ok "AUTOUPDATE=false landed in /data/coolify/source/.env — the platform will not move under its operators" \ + || no "AUTOUPDATE=false is NOT in coolify's .env — the pin is not holding" + cstate="$(docker inspect -f '{{.State.Status}}' coolify 2>/dev/null || echo absent)" + [ "$cstate" = running ] && ok "the coolify container is running" \ + || no "coolify container state: $cstate (expected running)" + leg "coolify install ($COOLIFY_VERSION)" \ + "$([ "$cstate" = running ] && echo "PASS ($(((SECONDS - t0) / 60)) min)" || echo "FAIL — container $cstate")" + else + no "coolify install FAILED — tail: $(tail -3 /tmp/drill-coolify.log | tr '\n' ' ')" + leg "coolify install ($COOLIFY_VERSION)" "FAIL — installer exited non-zero" + fi +fi + +# ============================================================================= +phase "Leg 2 — db dump/restore round-trip (test/db-integration.sh)" +# ============================================================================= +# Driven from the INSTALLED tree — the drill exercises what shipped, not the +# checkout this script happens to sit in. The leg's skip contract is the +# script's own (loud, reasoned, exit 0) and classify_leg keeps it a SKIP: +# counted, rendered distinctly, named in the record — never a pass. +db_out="$(mktemp)" +bash "$RIG_TREE/test/db-integration.sh" >"$db_out" 2>&1 +db_rc=$? +case "$(classify_leg "$db_rc" "$db_out")" in + pass) + db_numbers="$(tail -1 "$db_out")" + ok "db round-trip: $db_numbers" + leg "test/db-integration.sh" "PASS — $db_numbers" + ;; + skip) + db_reason="$(grep -m1 '^skip:' "$db_out")" + skip "db round-trip did not run — $db_reason" + leg "test/db-integration.sh" "SKIPPED — ${db_reason#skip: }" + ;; + fail) + no "db round-trip FAILED (exit $db_rc) — tail: $(tail -3 "$db_out" | tr '\n' ' ')" + leg "test/db-integration.sh" "FAIL — exit $db_rc" + ;; +esac +rm -f "$db_out" + +# ============================================================================= +phase "Leg 3 — runner lifecycle against a fork" +# ============================================================================= +# Register, take a job, deregister. The fork must carry a workflow_dispatch +# workflow (default drill.yml) whose job runs-on the 'drill' label — see +# drill/README.md. Tokens: RUNNER_TOKEN / RUNNER_REMOVE_TOKEN env, or minted +# via an authenticated gh. Without a fork or a token source the leg SKIPS, +# loudly, and the record says it did not run. +if [ -z "$RUNNER_REPO" ]; then + skip "runner lifecycle: no --runner-repo fork given — the leg did not run" + leg "runner lifecycle" "SKIPPED — no fork provided" +else + GH_OK=0 + command -v gh >/dev/null 2>&1 && gh auth status >/dev/null 2>&1 && GH_OK=1 + reg_token="${RUNNER_TOKEN:-}" + if [ -z "$reg_token" ] && [ "$GH_OK" -eq 1 ]; then + reg_token="$(gh api -X POST "repos/$RUNNER_REPO/actions/runners/registration-token" --jq .token 2>/dev/null)" + fi + if [ -z "$reg_token" ]; then + skip "runner lifecycle: no RUNNER_TOKEN and no authenticated gh to mint one — the leg did not run" + leg "runner lifecycle ($RUNNER_REPO)" "SKIPPED — no registration token source" + else + RUNNER_NAME="drill-$(hostname)-$$" + if RUNNER_TOKEN="$reg_token" run_logged /tmp/drill-runner-install.log \ + rig runner install --repo "$RUNNER_REPO" --name "$RUNNER_NAME" --labels drill; then + ok "rig runner install --repo $RUNNER_REPO exited 0 (registered as $RUNNER_NAME)" + else + no "runner install FAILED — tail: $(tail -3 /tmp/drill-runner-install.log | tr '\n' ' ')" + fi + rig runner status 2>/dev/null | grep -q "$RUNNER_REPO" \ + && ok "runner status names the fork: $RUNNER_REPO" \ + || no "runner status does not name $RUNNER_REPO" + + took_job=none + if [ "$GH_OK" -eq 1 ]; then + # Dispatch, then poll the newest run of that workflow to completion. + # ~5 min bound: a queued-forever run means the runner never picked it + # up, which is exactly what this check exists to catch. + if gh workflow run "$RUNNER_WORKFLOW" -R "$RUNNER_REPO" >/dev/null 2>&1; then + inf "dispatched $RUNNER_WORKFLOW on $RUNNER_REPO — waiting for the runner to take it (≤5 min)…" + took_job=timeout + for _i in $(seq 1 30); do + sleep 10 + run_json="$(gh run list -R "$RUNNER_REPO" --workflow "$RUNNER_WORKFLOW" --limit 1 --json status,conclusion 2>/dev/null)" + case "$run_json" in + *'"status":"completed"'*) + case "$run_json" in + *'"conclusion":"success"'*) took_job=success ;; + *) took_job=failed ;; + esac + break ;; + esac + done + else + took_job=nodispatch + fi + case "$took_job" in + success) ok "the runner took a job and it succeeded ($RUNNER_WORKFLOW)" ;; + failed) no "the dispatched job completed UNSUCCESSFULLY — the runner ran it, the workflow failed; read the run on $RUNNER_REPO" ;; + timeout) no "the dispatched job never completed within 5 min — the runner did not take it (is the workflow's runs-on label 'drill'?)" ;; + nodispatch) no "could not dispatch $RUNNER_WORKFLOW on $RUNNER_REPO — does the fork carry it, with workflow_dispatch? (see drill/README.md)" ;; + esac + else + skip "took a job: not attempted — no authenticated gh to dispatch $RUNNER_WORKFLOW with" + fi + + rem_token="${RUNNER_REMOVE_TOKEN:-}" + if [ -z "$rem_token" ] && [ "$GH_OK" -eq 1 ]; then + rem_token="$(gh api -X POST "repos/$RUNNER_REPO/actions/runners/remove-token" --jq .token 2>/dev/null)" + fi + if [ -n "$rem_token" ]; then + RUNNER_REMOVE_TOKEN="$rem_token" rig runner remove >/dev/null 2>&1 \ + && ok "rig runner remove deregistered cleanly" \ + || no "runner remove FAILED" + else + rig runner remove --local >/dev/null 2>&1 \ + && note "deregistered --local only (no removal token source) — delete the stale runner from $RUNNER_REPO's settings by hand" \ + || no "runner remove --local FAILED" + fi + rig runner status >/dev/null 2>&1 \ + && no "runner status still answers after remove — the deregistration did not take" \ + || ok "runner status confirms: nothing registered" + + leg "runner lifecycle ($RUNNER_REPO)" \ + "$(case "$took_job" in + success) echo "PASS — registered, took a job, deregistered clean" ;; + none) echo "PARTIAL — registered and deregistered; took a job: not attempted (no gh)" ;; + *) echo "FAIL — see Failed below" ;; + esac)" + fi +fi + +# ============================================================================= +phase "Summary" +# ============================================================================= +printf ' %s passed, %s failed, %s skipped\n' "$pass" "$fail" "$skipped" +if [ "${#findings[@]}" -gt 0 ]; then + echo + printf ' %s\n' "${findings[@]}" +fi + +mkdir -p "$(dirname "$RECORD")" +emit_record "$RECORD" +echo +inf "record written: $RECORD" +inf "commit it on the release branch as drills/$DRILL_VERSION.md — the" +inf "drill-recorded gate reads that file and nothing else (drills/README.md)." +[ "$fail" -eq 0 ] From 77cb4bdd64a7fba3307e410fd46f7231a4ab5a11 Mon Sep 17 00:00:00 2001 From: claude-bot-andresmgsl Date: Fri, 24 Jul 2026 00:31:53 +0000 Subject: [PATCH 2/4] test: the instrument's honesty, proven without hardware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test/drill.sh awk-extracts the harness's decision functions (the release.sh pattern) and drives them against fixtures: the ref refusal names both refs, a loud skip never classifies as a pass, the idempotence verdict is a real diff that goes non-empty when convergence is broken — demonstrated mechanically on every CI run — and the record emitter cannot produce a clean-sweep reading over a skipped leg. CI runs it in the check job. The tests caught three real harness bugs before any reviewer could: printf eating a '- '-leading format as options (a silently empty Failed section — the exact lie the record exists to prevent), tree_of trusting GNU readlink -f's exit 0 on a dangling final component, and the arg refusals sitting behind the root check in violation of the repo's own validated-before-root doctrine. (ceremony flow: issue #105) Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 5 + drill/drill.sh | 41 +++++--- test/drill.sh | 221 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 252 insertions(+), 15 deletions(-) create mode 100644 test/drill.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6734cf2..c412b37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,11 @@ jobs: run: bash test/cli.sh - name: release tests — rig's own surfaces run: bash test/release.sh + # The drill harness's honesty: refusals, the skip/pass/fail classifier, + # the idempotence capture-and-diff, the record emitter. Network-free and + # root-free — the live four-leg run is a release's drill, not CI's. + - name: drill harness tests — the instrument itself + run: bash test/drill.sh # The release guards, doctrine in heavy-duty/ceremony's README (#13's # conversion). Each one's war story — why it exists, what it refuses — # lives with its implementation upstream; the four pins below and the diff --git a/drill/drill.sh b/drill/drill.sh index 9323ff3..b460d15 100644 --- a/drill/drill.sh +++ b/drill/drill.sh @@ -121,7 +121,10 @@ run_logged() { tree_of() { local real real="$(readlink -f "$1" 2>/dev/null)" - [ -n "$real" ] || return 1 + # -e as well as -n: GNU readlink -f resolves a path whose LAST component + # does not exist (exit 0), so a dangling link would hand back a tree that + # is not there. + { [ -n "$real" ] && [ -e "$real" ]; } || return 1 dirname "$(dirname "$real")" } @@ -201,11 +204,12 @@ capture_state() { else printf ' (sshd absent)\n' fi - # Self's Tags/BackendState are the FIRST occurrences in the status JSON - # (Self serializes before Peer), and peers must not leak into the capture: - # another machine joining the tailnet between two captures is not a - # convergence diff on this one. - printf 'tailscale.self: %s\n' "$(tailscale status --json 2>/dev/null | tr -d '\n ' | grep -o '"BackendState":"[^"]*"\|"Tags":\[[^]]*\]' | head -n2 | tr '\n' ' ' || true)" + # Self's Tags is the FIRST occurrence in the status JSON (Self serializes + # before Peer). Tags only, nothing livelier: peers joining, IPs renewing + # or a backend-state flap between two captures is not a convergence diff + # on this box, and a capture that can move on its own poisons the + # idempotence verdict with noise. + printf 'tailscale.self.tags: %s\n' "$(tailscale status --json 2>/dev/null | tr -d '\n ' | grep -o '"Tags":\[[^]]*\]' | head -n1 || true)" printf 'users-ledger:\n' sed 's/^/ /' "$ledger" 2>/dev/null || printf ' (absent)\n' # Per-operator effective state: the account, its groups, its lock state, @@ -271,13 +275,16 @@ emit_record() { if [ "$fail" -eq 0 ] && [ "$skipped" -eq 0 ]; then printf '\nFailed: nothing. Every leg ran and every check passed.\n' else + # printf --: a format opening with '- ' reads as an option to bash's + # printf and emits NOTHING — a record whose Failed section silently + # vanished is exactly the lie this file exists to make impossible. [ "$fail" -gt 0 ] && printf '\nFailed:\n' for line in "${findings[@]:-}"; do - case "$line" in FAIL:*) printf '- %s\n' "$line" ;; esac + case "$line" in FAIL:*) printf -- '- %s\n' "$line" ;; esac done [ "$skipped" -gt 0 ] && printf '\nSkipped — these did NOT run, and this record is not evidence for them:\n' for line in "${findings[@]:-}"; do - case "$line" in SKIP:*) printf '- %s\n' "$line" ;; esac + case "$line" in SKIP:*) printf -- '- %s\n' "$line" ;; esac done fi printf '\nThe isolation boundary was NOT asserted here: it is box'\''s drill'\''s\n' @@ -287,11 +294,10 @@ emit_record() { # ============================================================================= # Pre-flight — every refusal this run can see coming fires here, before -# anything is installed or any credential is spent (repo doctrine: errors -# belong at the top of the run). +# anything is installed or any credential is spent. Args are validated BEFORE +# the root check (repo doctrine, bootstrap.sh:114 — so the refusals are +# testable without root, and a typo costs a re-type, never a re-ssh). # ============================================================================= -[ "$(id -u)" -eq 0 ] || { echo "drill: must run as root (bootstrap, runner, coolify and db all require it) — ssh in as root on the throwaway machine" >&2; exit 1; } - # Both refs EXPLICIT, or nothing runs. Defaulting either to main is exactly # the #103 hazard this harness exists to refuse: "I drilled the release" must # not quietly mean "I drilled whatever main was that afternoon". @@ -313,6 +319,8 @@ if [ -z "$USERS_FILE" ]; then fi [ -r "$USERS_FILE" ] || { echo "drill: cannot read users file: $USERS_FILE" >&2; exit 2; } +[ "$(id -u)" -eq 0 ] || { echo "drill: must run as root (bootstrap, runner, coolify and db all require it) — ssh in as root on the throwaway machine" >&2; exit 1; } + # The tailnet join needs a key unless this machine already joined (a re-drill # on the same throwaway). Caught here, not 10 apt-minutes into bootstrap. if [ -z "${TS_AUTHKEY:-}" ]; then @@ -416,7 +424,10 @@ if [ "$BOOTSTRAP_OK" -eq 1 ]; then uhome="$(getent passwd "$u" | cut -d: -f6)" [ -s "$uhome/.ssh/authorized_keys" ] || users_bad="$users_bad $u(no-keys)" done < <(cat "${DRILL_LEDGER:-/etc/rig/users}" 2>/dev/null) - n_users="$(grep -c ' active$' "${DRILL_LEDGER:-/etc/rig/users}" 2>/dev/null || echo 0)" + # NOT 'grep -c … || echo 0': grep -c already prints 0 on no match (and then + # exits 1), so the fallback would emit a second line into the substitution. + n_users="$(grep -c ' active$' "${DRILL_LEDGER:-/etc/rig/users}" 2>/dev/null)" || true + n_users="${n_users:-0}" [ -z "$users_bad" ] && [ "$n_users" -gt 0 ] \ && ok "operators converged: $n_users active, accounts and keys present" \ || no "operators NOT converged:${users_bad:- ledger empty}" @@ -472,11 +483,11 @@ case "$MARKER_LINE" in ok "installed box confirms: $BOXREPO@$BOXREF" if box doctor >/dev/null 2>&1; then ok "box doctor passes — setup-host converged; the host stack stands (box's own effective-state verdict)" + leg "--host yes: pinned box installed, host stack up" "PASS — $BOXREPO@$BOXREF, box doctor clean" else no "box is installed but 'box doctor' does not pass — the host stack is unproven (run 'box doctor' for box's verdict)" + leg "--host yes: pinned box installed, host stack up" "FAIL — box doctor does not pass" fi - leg "--host yes: pinned box installed, host stack up" \ - "$(box doctor >/dev/null 2>&1 && echo "PASS — $BOXREPO@$BOXREF, box doctor clean" || echo "FAIL — box doctor does not pass")" else no "no 'box' on PATH after a host=yes bootstrap — the box install did not take (bootstrap warns rather than dies there; the drill does not)" leg "--host yes: pinned box installed, host stack up" "FAIL — box CLI never landed" diff --git a/test/drill.sh b/test/drill.sh new file mode 100644 index 0000000..dbb6477 --- /dev/null +++ b/test/drill.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +# test/drill.sh — the drill harness's HONESTY, proven without hardware. +# +# drill/drill.sh is the instrument (#105), so what this suite tests is the +# instrument itself: the refusals, the classifications, the capture-and-diff +# that decides idempotence, and the record emitter — the parts whose lies +# would be believed, months later, by a reader of drills/.md. The +# four-leg live run on a real Debian machine is #107's exercise, not this +# file's: nothing here needs root, Docker, a tailnet or the network. +# +# Extraction pattern is test/release.sh's: the functions under test are +# awk-extracted from drill/drill.sh and driven against fixtures, so the tests +# exercise the shipped bytes, and the extraction check itself guards the awk +# against a drifted function boundary. +# Deliberately no `set -e` — the harness asserts on failing commands. +set -u +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" || exit 1 +PASS=0 FAIL=0 + +# check +check() { + local desc="$1" want="$2" substr="$3"; shift 3 + local out rc + out="$("$@" 2>&1)"; rc=$? + if [ "$rc" -ne "$want" ]; then + echo "FAIL: $desc — exit $rc, wanted $want" + printf '%s\n' "$out" | sed 's/^/ /' + FAIL=$((FAIL + 1)); return + fi + if [ -n "$substr" ] && ! printf '%s' "$out" | grep -qF -e "$substr"; then + echo "FAIL: $desc — output missing '$substr'" + printf '%s\n' "$out" | sed 's/^/ /' + FAIL=$((FAIL + 1)); return + fi + echo "ok: $desc"; PASS=$((PASS + 1)) +} + +# refute — the file must NOT contain the substring. +refute() { + if grep -qF -e "$2" "$3"; then + echo "FAIL: $1 — found forbidden '$2'" + FAIL=$((FAIL + 1)); return + fi + echo "ok: $1"; PASS=$((PASS + 1)) +} + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +# --- the functions under test, extracted ------------------------------------- +FNS="$WORK/drill-fns.sh" +for fn in tree_of assert_installed_from classify_leg capture_state emit_record; do + awk "/^${fn}\(\) \{/,/^\}/" "$ROOT/drill/drill.sh" >> "$FNS" +done +for fn in tree_of assert_installed_from classify_leg capture_state emit_record; do + check "extraction guards the awk: ${fn}() landed" 0 "${fn}() {" grep -F "${fn}() {" "$FNS" +done +# shellcheck source=/dev/null +. "$FNS" + +# ============================================================================= +# tree_of — the versioned tree behind a CLI's symlink chain +# ============================================================================= +IR="$WORK/install"; mkdir -p "$IR/versions/1.2.3/bin" +: > "$IR/versions/1.2.3/bin/rig" +ln -s "versions/1.2.3" "$IR/current" +mkdir -p "$WORK/bin" +ln -s "$IR/current/bin/rig" "$WORK/bin/rig" +check "tree_of resolves a current-symlink chain to versions/" 0 "$IR/versions/1.2.3" \ + tree_of "$WORK/bin/rig" +ln -s "$IR/gone/bin/rig" "$WORK/bin/dangling" +check "tree_of refuses a dangling chain — a tree that is not there is not a tree" 1 "" \ + tree_of "$WORK/bin/dangling" + +# ============================================================================= +# assert_installed_from — the up-front ref refusal, naming both refs +# ============================================================================= +TREE="$WORK/tree-main"; mkdir -p "$TREE" +printf 'heavy-duty/rig@main\n' > "$TREE/INSTALLED_FROM" +check "matching INSTALLED_FROM passes silently" 0 "" \ + assert_installed_from rig "$TREE" "heavy-duty/rig@main" +check "a mismatch refuses (the #103 hazard: asked release, got main)" 1 "FATAL" \ + assert_installed_from rig "$TREE" "heavy-duty/rig@release/9.9.9" +check "…the refusal names the ref that was ASKED for" 1 "heavy-duty/rig@release/9.9.9" \ + assert_installed_from rig "$TREE" "heavy-duty/rig@release/9.9.9" +check "…and the ref that actually LANDED" 1 "heavy-duty/rig@main" \ + assert_installed_from rig "$TREE" "heavy-duty/rig@release/9.9.9" +check "an unreadable INSTALLED_FROM refuses too — absence is not a match" 1 "" \ + assert_installed_from rig "$WORK/no-such-tree" "heavy-duty/rig@main" + +# ============================================================================= +# classify_leg — a loud skip is a SKIP, never a pass (box#153's defect class) +# ============================================================================= +printf 'skip: docker not installed — nothing to exercise\n' > "$WORK/out-skip" +printf 'ok: seeded\nok: restored\n---\n14 passed, 0 failed\n' > "$WORK/out-pass" +printf 'FAIL: restore blew up\n' > "$WORK/out-fail" +check "exit 0 + 'skip:' line classifies as skip" 0 "skip" classify_leg 0 "$WORK/out-skip" +check "exit 0, no skip line, classifies as pass" 0 "pass" classify_leg 0 "$WORK/out-pass" +check "non-zero exit classifies as fail" 0 "fail" classify_leg 1 "$WORK/out-fail" +check "a skip line cannot rescue a non-zero exit (fail wins)" 0 "fail" \ + classify_leg 1 "$WORK/out-skip" + +# ============================================================================= +# capture_state + diff — the idempotence verdict's machinery. The claim in +# #105's acceptance criteria: the assertion is a REAL diff of captured state, +# and it FAILS when convergence is broken — demonstrated here, mechanically, +# on every CI run, by breaking the state between two captures. +# ============================================================================= +FIX="$WORK/fix"; mkdir -p "$FIX/sudoers.d" +printf 'role=staging-server root-door=open host=yes join=authkey\n' > "$FIX/role" +printf 'schema=1\nbootstrapped_by=9.9.9\nbootstrapped_at=T\nconverged_by=9.9.9\nconverged_at=T\n' > "$FIX/manifest" +printf 'dan active\nghost revoked\n' > "$FIX/ledger" +printf 'APT::Periodic::Update-Package-Lists "1";\n' > "$FIX/autoup" +printf '127.0.0.1 localhost\n127.0.1.1\tstaging-server\n' > "$FIX/hosts" +printf 'nosuchdrilluser ALL=(ALL) NOPASSWD:ALL\n' > "$FIX/sudoers.d/00-rig-nosuch" +# A stubbed sshd, so the effective-config section is exercised rather than +# skipped on a box with no daemon (repo precedent: test/release.sh's curl). +STUB="$WORK/stub"; mkdir -p "$STUB" +# The single-quoted $SSHD_FIXTURE is the STUB's expansion, not this shell's. +# shellcheck disable=SC2016 +printf '#!/usr/bin/env bash\ncat "$SSHD_FIXTURE"\n' > "$STUB/sshd"; chmod +x "$STUB/sshd" +printf 'passwordauthentication no\npermitrootlogin prohibit-password\n' > "$FIX/sshd-T" + +cap() { # cap — capture_state against the fixture set + RIG_ROLE_MARKER="$FIX/role" RIG_MANIFEST="$FIX/manifest" \ + DRILL_LEDGER="$FIX/ledger" DRILL_AUTOUPGRADES="$FIX/autoup" \ + DRILL_ETC_HOSTS="$FIX/hosts" DRILL_SUDOERS_DIR="$FIX/sudoers.d" \ + SSHD_FIXTURE="$FIX/sshd-T" PATH="$STUB:$PATH" \ + bash -c '. "$1"; capture_state "$2"' _ "$FNS" "$2" 2>/dev/null + : +} +# cap runs capture_state in a child bash so the PATH stub cannot leak into +# this harness; $2 arrives as the capture's outfile. +cap out "$WORK/cap1" +cap out "$WORK/cap2" +check "two captures over untouched state diff EMPTY (the converged verdict)" 0 "" \ + diff -u "$WORK/cap1" "$WORK/cap2" +check "the capture reads the fixtures, not the machine (marker line present)" 0 "role=staging-server" \ + grep -o 'role=staging-server[^"]*' "$WORK/cap1" +check "…the sshd section captured the effective config" 0 "passwordauthentication no" \ + cat "$WORK/cap1" +check "…a ledger user with no account reads as one, deterministically" 0 "(no account)" \ + cat "$WORK/cap1" + +# Break convergence: the re-run "changed" the role marker and root's door. +printf 'role=staging-server root-door=closed host=yes join=authkey\n' > "$FIX/role" +printf 'passwordauthentication yes\npermitrootlogin prohibit-password\n' > "$FIX/sshd-T" +cap out "$WORK/cap3" +check "a broken convergence makes the diff NON-empty — the assertion can fail" 1 "root-door=closed" \ + diff -u "$WORK/cap1" "$WORK/cap3" +check "…and the diff names the drifted sshd keyword, not just 'differs'" 1 "passwordauthentication yes" \ + diff -u "$WORK/cap1" "$WORK/cap3" + +# ============================================================================= +# emit_record — the record is drills/README.md's shape, and it cannot lie: +# a failed run still emits, a skipped leg is named, no clean-sweep reading. +# ============================================================================= +emit() { # emit — emit_record with the harness globals staged + DRILL_VERSION="9.9.9" RUN_ID="drill-2026-01-01-a" \ + REF="release/9.9.9" BOXREF="release/0.4.0" RIG_SHA="5d6e7f8" BOX_SHA="1a2b3c4" \ + bash -c ' + . "$1" + pass=12 fail=1 skipped=1 + findings=("FAIL: coolify container state: absent" "SKIP: runner lifecycle: no --runner-repo fork given — the leg did not run" "NOTE: something worth a line") + LEG_NAMES=("convergence — bootstrap staging-server reaches its role" "re-converge (idempotence)" "coolify install (4.1.2)" "runner lifecycle") + LEG_RESULTS=("PASS (312s)" "clean, no changes" "FAIL — container absent" "SKIPPED — no fork provided") + emit_record "$2" + ' _ "$FNS" "$2" +} +emit out "$WORK/record.md" +check "record: the version-and-date heading" 0 "# Release drill — 9.9.9 — " head -1 "$WORK/record.md" +check "record: the run ID that joins the family's records" 0 "Run ID: drill-2026-01-01-a" cat "$WORK/record.md" +check "record: both pinned refs with their SHAs" 0 "rig@5d6e7f8 (RIG_REF=release/9.9.9)" cat "$WORK/record.md" +check "record: …box's too" 0 "box@1a2b3c4 (BOX_REF=release/0.4.0)" cat "$WORK/record.md" +check "record: one table row per leg, result verbatim" 0 "| re-converge (idempotence) | clean, no changes |" cat "$WORK/record.md" +check "record: the numbers, skips counted apart from passes" 0 "12 passed, 1 failed, 1 skipped" cat "$WORK/record.md" +check "record: a FAILED run still names what failed (evidence, not success)" 0 "FAIL: coolify container state: absent" cat "$WORK/record.md" +check "record: a skipped leg is stated as NOT run, by name" 0 "SKIP: runner lifecycle" cat "$WORK/record.md" +check "record: the skip section says the record is not evidence for it" 0 "not evidence" cat "$WORK/record.md" +check "record: the isolation boundary is named as box's, in words" 0 "NOT asserted here" cat "$WORK/record.md" +refute "record with a skip cannot read as a clean sweep" "Failed: nothing" "$WORK/record.md" +refute "notes are findings for the log, not failures for the record" "NOTE: something" "$WORK/record.md" + +# The all-green shape: says so plainly, and only then. +DRILL_VERSION="9.9.9" RUN_ID="drill-2026-01-01-a" \ +REF="release/9.9.9" BOXREF="release/0.4.0" RIG_SHA="5d6e7f8" BOX_SHA="1a2b3c4" \ +bash -c ' + . "$1" + pass=20 fail=0 skipped=0 + findings=() + LEG_NAMES=("convergence" "re-converge (idempotence)") + LEG_RESULTS=("PASS" "clean, no changes") + emit_record "$2" +' _ "$FNS" "$WORK/record-green.md" +check "an all-green record says every leg ran and passed" 0 "Every leg ran and every check passed" \ + cat "$WORK/record-green.md" + +# ============================================================================= +# the shipped script itself +# ============================================================================= +# Arg refusals fire before the root check (repo doctrine, bootstrap.sh:114), +# which is what makes them provable here without a throwaway machine. +check "drill.sh refuses to run without BOTH refs pinned (#103)" 2 "--box-ref" \ + env -u RIG_REF -u BOX_REF bash "$ROOT/drill/drill.sh" --rig-ref release/9.9.9 --yes +check "…and the refusal shows which ref is missing" 2 "" \ + env -u RIG_REF -u BOX_REF bash "$ROOT/drill/drill.sh" --rig-ref release/9.9.9 --yes +check "a tenant role is refused — the drill converges machines, not guests" 2 "not a machine role" \ + bash "$ROOT/drill/drill.sh" --rig-ref r --box-ref b --role claude-box --yes +check "no --users is a refusal, naming why the drill will not default it" 2 "--users is required" \ + bash "$ROOT/drill/drill.sh" --rig-ref r --box-ref b --yes +check "an unreadable users file dies before anything is spent" 2 "cannot read users file" \ + bash "$ROOT/drill/drill.sh" --rig-ref r --box-ref b --users "$WORK/no-such-users" --yes +check "an unknown flag dies loudly, exit 2" 2 "unknown option" \ + bash "$ROOT/drill/drill.sh" --frobnicate +check "--help prints the header and exits 0" 0 "THROWAWAY" \ + bash "$ROOT/drill/drill.sh" --help + +echo "---" +echo "$PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ] From 5bcd8853d47a835546620b75ed9841762d47deda Mon Sep 17 00:00:00 2001 From: claude-bot-andresmgsl Date: Fri, 24 Jul 2026 00:34:30 +0000 Subject: [PATCH 3/4] docs: the run is written down, and the doctrine stops claiming there is no instrument drill/README.md is the repeatable procedure #107's second checkbox asks for: prerequisites (the throwaway machine, the tagged key, the fork's drill workflow, the pins), the invocation, what each leg asserts, where the record lands. drills/README.md's harness disclaimer flips to point at the instrument, its legs list and example record match what drill.sh actually runs and emits, and CONTRIBUTING's drill sentence names the script. Changelog entry under Unreleased. (ceremony flow: issue #105) Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 1 + CONTRIBUTING.md | 3 +- drill/README.md | 91 ++++++++++++++++++++++++++++++++++++++++++++++++ drills/README.md | 29 ++++++++------- 4 files changed, 111 insertions(+), 13 deletions(-) create mode 100644 drill/README.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 7063c93..7296571 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ on the way to cutting its first release, and this file starts there. ### Added +- `drill/drill.sh` — the drill has an instrument: pinned-ref assertion, a mechanical idempotence diff, and a `drills/.md` record emitter (#105) - `kimi-box` joins the box tenant roles — the Kimi CLI agent guest (#109) - The `changelog-armed` guard returns, version-keyed (#112, ceremony#13) - The `.ceremony/` doctrine mirror, verified by `docs-sync` on every PR (#112, ceremony#19) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e0cf609..a7afb1a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -72,7 +72,8 @@ Bare `X.Y.Z` tags, no `v`; the tag's source tarball is the package `install.sh` downloads — rig ships no other artifact. What stays rig's is the **drill** — the real-hardware gate before the -handoff of a release PR: tenant guests minted and converged via box, +handoff of a release PR, run by `drill/drill.sh` (#105): `rig bootstrap` +converging the machine to its role twice with the second run diffed empty, `test/db-integration.sh`, the runner lifecycle against a fork, a coolify install. Rig's drill asserts **convergence** (a machine reaches its role, idempotently), it runs `--host yes` with `BOX_REF=release/` so diff --git a/drill/README.md b/drill/README.md new file mode 100644 index 0000000..bf0004a --- /dev/null +++ b/drill/README.md @@ -0,0 +1,91 @@ +# The drill — running it + +`drill/drill.sh` is the instrument; `drills/` is the record it feeds +(see [drills/README.md](../drills/README.md) for what a record means and +how the three repos' drills relate). rig's drill asserts **convergence**: +a machine reaches its role, idempotently. This file is the procedure — +written down so a run is repeatable, not reconstructed from memory each +release (#105, and #107's debt). + +## What you need + +- **A throwaway Debian 13 machine** you can format, reached as root. The + drill hardens its sshd, renames it, joins it to a tailnet, and installs + box/Incus, Coolify and a GitHub runner on it. It is not coming back. + The machine is its own reset — there is no teardown script and no need + for one. +- **The pinned candidate refs, both of them.** `--rig-ref` and + `--box-ref` are required; the harness refuses to run without them and + refuses to continue if what installed disagrees with what was asked + (`INSTALLED_FROM`, both trees). Until heavy-duty/rig#103 lands, both + installers default to `main` when unpinned — which is exactly why the + drill will not let a ref go unstated. +- **A single-use, tagged tailscale pre-auth key** in `TS_AUTHKEY` + (`tag:local` for the default `staging-server` role — bootstrap refuses + `tag:server` outside the control-plane shapes). +- **A users file** (`--users`) naming at least one operator — leg 1 + asserts the accounts and keys actually converged. +- **For leg 3** (runner lifecycle): a fork to register against + (`--runner-repo you/rig`) carrying a `workflow_dispatch` workflow — + default name `drill.yml` — whose job has `runs-on: [self-hosted, drill]` + and does something trivial (`echo drilled`). Tokens come from an + authenticated `gh`, or from `RUNNER_TOKEN` / `RUNNER_REMOVE_TOKEN`. + Without a fork the leg **skips, loudly, into the record**. +- **For leg 4** (coolify): a version pin, `--coolify-version 4.1.2`. + No pin, no leg — rig's own `coolify install` refuses to default a + version and so does its drill. The skip is recorded. +- **A run ID** (`--run-id`) when this drill shares a substrate with + box's or cast's — the shared ID is what lets the per-repo records be + joined afterwards. Defaults to `drill-`. + +## Running it + +From a checkout of this repo on the throwaway machine (the record lands +in the checkout's `drills/`): + +```sh +TS_AUTHKEY=tskey-... bash drill/drill.sh \ + --rig-ref release/0.4.0 --box-ref release/0.10.0 \ + --users ./drill-users --run-id drill-2026-07-24-a \ + --coolify-version 4.1.2 --runner-repo you/rig --yes +``` + +It runs unattended from there. Legs execute as 1, 4, 2, 3 — Coolify's +installer is what puts Docker on the box and the db leg needs a daemon — +and the record lists them as they ran. A failing check never aborts the +run (`set -u`, no `-e`: a failing check is data), and the summary counts +passes, failures and skips separately. + +## What it asserts + +1. **Convergence, and idempotence.** `rig bootstrap --users …` + reaches the declared role, asserted on *effective* state — the marker, + `sshd -T`, the granted tailnet tag, the operators' accounts and keys. + Then bootstrap runs **again**, and the state captured before and after + the re-run must diff **empty**. The diff is mechanical; "watched it + not obviously break" is exactly what this leg exists to replace. + Riding along, the `--host yes` assertions: the **pinned** box + installed (`INSTALLED_FROM` matches `--box-ref`, fatal if not), + `box doctor` passes. It stops there and says so in the output — the + isolation boundary is **box's** drill's assertion, never rig's. +2. **db** — `test/db-integration.sh` from the *installed* tree: a real + dump/restore round-trip. Its clean-skip contract (no Docker → loud + skip, exit 0) survives into the record as a SKIP, never a pass. +3. **Runner lifecycle** — register against the fork, dispatch the drill + workflow and watch the runner take it, deregister, and assert the + box's registration is actually gone. +4. **Coolify** — installed at the pin, `AUTOUPDATE=false` landed in the + effective `.env`, container running. + +## The record + +The run always ends by writing `drills/.md` (the version is the +installed tree's own `VERSION`) — on failures too: **a failed drill is a +valid record**; the gate wants evidence, not success. Skipped legs are +named as not-run so the record can never read as a clean sweep. Commit +the file on the release branch; the `drill-recorded` guard reads that +file and nothing else. + +The instrument's own honesty — the refusals, the skip accounting, the +capture-and-diff, the emitter — is `test/drill.sh`'s job, and CI runs it +on every PR. The live four-leg run is a release's job, once per cycle. diff --git a/drills/README.md b/drills/README.md index 3a8af5b..3928494 100644 --- a/drills/README.md +++ b/drills/README.md @@ -15,12 +15,13 @@ The directory is `drills/`, not `.drills/` — a dot-directory is invisible to any glob without `dotglob`, which is how #70 here and box#116 / box#118 all happened. -**This directory is the record, not the instrument.** rig has **no drill -harness script of its own**; its legs are run by following the documented -procedure, and the harness lives in heavy-duty/box's `drill/`. rig does not -reach into it to decide whether rig may ship: a cross-repo lookup that fails -silently degrades to "pass", which is the UNREADABLE-vs-NONE shape #90 fixed. -The gate reads a file in this repo, and nothing else. +**This directory is the record, not the instrument.** The instrument is +[`drill/drill.sh`](../drill/README.md) (#105): it runs the legs, asserts the +pinned refs actually landed, decides idempotence by a mechanical state diff, +and emits the record file this directory holds. rig does not reach into +another repo's harness to decide whether rig may ship: a cross-repo lookup +that fails silently degrades to "pass", which is the UNREADABLE-vs-NONE shape +#90 fixed. The gate reads a file in this repo, and nothing else. ## What the gate requires @@ -40,13 +41,16 @@ not success. ## The drill -rig's legs: +rig's legs (#105; `drill/drill.sh` runs them): -- tenant guests minted and converged **via box** +- `rig bootstrap ` converges the machine to its role — then runs + **again**, and the captured state must diff **empty** (idempotence, + decided mechanically). On a host=yes role this is also what installs the + pinned box and asserts its host stack stands. - `bash test/db-integration.sh` against a real Postgres on the machine - the GitHub runner lifecycle — register, take a job, deregister — against a fork -- a coolify install +- a coolify install, pinned, `AUTOUPDATE=false` box and rig are **mutually recursive**: `rig bootstrap --host yes` installs box and runs box's `setup-host`, while box's guests converge back through rig's @@ -100,11 +104,12 @@ Candidate refs: box@1a2b3c4 (BOX_REF=release/0.4.0), rig@5d6e7f8, cast@9a0b1c2. | Leg | Result | | --- | --- | -| tenant guests minted + converged via box | 3/3 | +| convergence — bootstrap staging-server reaches its role | PASS (312s) | | re-converge (idempotence) | clean, no changes | -| `test/db-integration.sh` | 14/14 | +| --host yes: pinned box installed, host stack up | PASS — box doctor clean | +| `test/db-integration.sh` | PASS — 14 passed, 0 failed | | runner lifecycle against a fork | PASS — registered, took a job, deregistered clean | -| coolify install | PASS, ~6 min | +| coolify install (4.1.2) | PASS (6 min) | Failed: `rig users apply` left one revoked key in `authorized_keys` (filed #NNN). Everything else clean. From 7b2de4a9e6d307804d64e077c221230f5b9f7bf7 Mon Sep 17 00:00:00 2001 From: claude-bot-andresmgsl Date: Fri, 24 Jul 2026 00:36:00 +0000 Subject: [PATCH 4/4] fix: the job poll cannot mistake an old run for the dispatched one, and --help covers its own header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner leg reads the newest run ID before dispatching and only judges a run with a different ID — workflow_dispatch takes seconds to materialize a run, and the previous run's 'completed' was one poll away from being read as ours. --help's sed range stops where the header does. (ceremony flow: issue #105) Co-Authored-By: Claude Fable 5 --- drill/drill.sh | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/drill/drill.sh b/drill/drill.sh index b460d15..8e10e89 100644 --- a/drill/drill.sh +++ b/drill/drill.sh @@ -74,7 +74,7 @@ while [ $# -gt 0 ]; do --coolify-version) COOLIFY_VERSION="$2"; shift 2 ;; --runner-repo) RUNNER_REPO="$2"; shift 2 ;; --runner-workflow) RUNNER_WORKFLOW="$2"; shift 2 ;; - -h|--help) sed -n '2,36p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + -h|--help) sed -n '2,33p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; *) echo "drill: unknown option: $1 (see --help)" >&2; exit 2 ;; esac done @@ -593,22 +593,30 @@ else took_job=none if [ "$GH_OK" -eq 1 ]; then # Dispatch, then poll the newest run of that workflow to completion. - # ~5 min bound: a queued-forever run means the runner never picked it - # up, which is exactly what this check exists to catch. + # The newest run's ID is read BEFORE dispatching, so an old completed + # run can never be mistaken for the one just dispatched (the poll's + # verdict must be about OUR run, and workflow_dispatch takes a few + # seconds to materialize a run at all). ~5 min bound: a queued-forever + # run means the runner never picked the job up, which is exactly what + # this check exists to catch. + pre_id="$(gh run list -R "$RUNNER_REPO" --workflow "$RUNNER_WORKFLOW" --limit 1 --json databaseId --jq '.[0].databaseId' 2>/dev/null)" if gh workflow run "$RUNNER_WORKFLOW" -R "$RUNNER_REPO" >/dev/null 2>&1; then inf "dispatched $RUNNER_WORKFLOW on $RUNNER_REPO — waiting for the runner to take it (≤5 min)…" took_job=timeout for _i in $(seq 1 30); do sleep 10 - run_json="$(gh run list -R "$RUNNER_REPO" --workflow "$RUNNER_WORKFLOW" --limit 1 --json status,conclusion 2>/dev/null)" - case "$run_json" in - *'"status":"completed"'*) - case "$run_json" in - *'"conclusion":"success"'*) took_job=success ;; - *) took_job=failed ;; - esac - break ;; - esac + run_line="$(gh run list -R "$RUNNER_REPO" --workflow "$RUNNER_WORKFLOW" --limit 1 \ + --json databaseId,status,conclusion --jq '.[0] | "\(.databaseId) \(.status) \(.conclusion)"' 2>/dev/null)" + read -r rid rstatus rconc <<< "$run_line" + [ -n "${rid:-}" ] || continue + [ "$rid" != "${pre_id:-}" ] || continue + if [ "${rstatus:-}" = completed ]; then + case "${rconc:-}" in + success) took_job=success ;; + *) took_job=failed ;; + esac + break + fi done else took_job=nodispatch