fix: sanitize the console dump (no more scrambled terminal) + tear down failed mints

Two bugs surfaced running the 0.5.0 drill with --purge-storage on a
cold btrfs pool:

1. wait_agent dumped the VM's RAW console log on timeout — full of
   terminal escape sequences and a firmware menu — which scrambled the
   operator's terminal, and doubly so when it landed in a log they were
   tail -f'ing ('*Debian GNU/Linux', 'ESC to return previous menu',
   ^[^[^[). Now: capture to /tmp/box-console-<n>.log, strip everything
   but printable ASCII + tab/newline, print only a short sanitized tail.
   Nothing raw reaches a terminal.

2. A failed mint left its stuck VM running, starving the NEXT box's boot
   and cascading more 5-min timeouts (tpl failed → codex failed). Every
   mint-failure branch now tears the box down before continuing.
This commit is contained in:
claude-hdb 2026-07-14 16:38:54 +00:00
parent 5deef69621
commit c4cc9f43d1
2 changed files with 17 additions and 4 deletions

17
bin/box
View file

@ -500,14 +500,23 @@ pick_mode() {
# forensics: the VM's console says why, and the box is torn down by whoever
# called us before anyone can read it.
wait_agent() {
local n="$1" i
local n="$1" i clog
echo "box: waiting for instance agent..."
for i in $(seq 1 150); do
if incus exec "$n" -- true </dev/null >/dev/null 2>&1; then return; fi
if [ "$i" -eq 150 ]; then
echo "box: instance agent never came up. The VM's console log:" >&2
timeout -k 5 15 incus console "$n" --show-log 2>/dev/null | tail -15 | sed 's/^/ /' >&2
die "agent unreachable after 5 minutes (incus console $n to inspect live)"
# The console log is FULL of terminal escape sequences (boot messages,
# a firmware menu). Dumping it raw scrambles the operator's terminal —
# and doubly so when it lands in a log someone is tail -f'ing. Capture
# it to a file, STRIP everything but printable ASCII + tab/newline, and
# print only a short sanitized tail. Nothing raw ever reaches a terminal.
clog="/tmp/box-console-$n.log"
timeout -k 5 15 incus console "$n" --show-log </dev/null >"$clog.raw" 2>/dev/null || true
tr -cd '\11\12\40-\176' <"$clog.raw" >"$clog" 2>/dev/null; rm -f "$clog.raw"
echo "box: instance agent never came up after 5 minutes." >&2
echo "box: sanitized console log → $clog (last lines:)" >&2
tail -8 "$clog" 2>/dev/null | sed 's/^/ /' >&2
die "agent unreachable (inspect live: incus console $n)"
fi
sleep 2
done

View file

@ -429,6 +429,8 @@ if mint_box /tmp/mint-tpl.log --name tpl; then
box rm tpl --force >/dev/null 2>&1 && ok "blank box removed" || no "could not remove the blank box"
else
no "blank mint FAILED — tail: $(tail -3 /tmp/mint-tpl.log | tr '\n' ' ')"
# Tear the stuck box down — a failed mint that lingers starves the next one.
timeout -k 5 60 incus delete -f tpl >/dev/null 2>&1
fi
# The generic mechanic (metadata, placement, user, isolation parity) is proven
@ -450,6 +452,7 @@ for t in codex grok; do
box rm "$t" --force >/dev/null 2>&1 && ok "$t box removed" || no "$t: could not remove"
else
no "$t mint FAILED — tail: $(tail -3 "/tmp/mint-$t.log" | tr '\n' ' ')"
timeout -k 5 60 incus delete -f "$t" >/dev/null 2>&1
fi
done
@ -459,6 +462,7 @@ if mint_box /tmp/mint-drill.log --name drill --template claude; then
ok "box new --name drill --template claude ($((SECONDS - t0))s)"
else
no "box new FAILED — tail: $(tail -3 /tmp/mint-drill.log | tr '\n' ' ')"
timeout -k 5 60 incus delete -f drill >/dev/null 2>&1
echo; echo "── cannot continue without a box"; printf ' %s\n' "${findings[@]}"; exit 1
fi