From 28be7a0ec78c36a1b77bd4bfac9672a3e2cde484 Mon Sep 17 00:00:00 2001 From: claude-hdb Date: Tue, 14 Jul 2026 00:55:39 +0000 Subject: [PATCH] fix: a failed cold mint must say why, and the doctor must find the cause MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two cold mints in a row died with cloud-init 'status: error' on a host the doctor had just certified clean — so the earlier "leftover mutations poisoned the network" theory is dead, and the DNS failure is reproducible rather than transient. 'claudebox new' printed four hundred dots and the word "error", leaving the user with nothing to act on: the reason was in the box's own log and nobody was told the log existed. It now prints cloud-init's status, the fetch/resolve errors from the box's log, and how to inspect the box — which is left running, because a box that failed to build is evidence, not garbage. It also names the usual culprit: the host's resolver. doctor.sh gains the diagnosis that keeps being done by hand: · what the HOST resolves through, and whether that is a CGNAT/Tailscale resolver the boxes inherit (issue #33); · whether claudenet's resolver is pinned; · and inside a box, the question that settles it — DNS is broken, but can it still reach 1.1.1.1 BY ADDRESS? If yes, egress is fine and the fault is purely the inherited forwarder. · --pin-dns applies the #33 fix (raw.dnsmasq: no-resolv + public servers) so the hypothesis can be TESTED rather than argued. Co-Authored-By: Claude Fable 5 --- bin/claudebox | 21 ++++++++++++++- drill/doctor.sh | 70 +++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 79 insertions(+), 12 deletions(-) diff --git a/bin/claudebox b/bin/claudebox index 96a3771..de9a320 100755 --- a/bin/claudebox +++ b/bin/claudebox @@ -457,7 +457,26 @@ cmd_new() { "${extra[@]}" wait_agent "$instance" echo "claudebox: waiting for phase-1 (cloud-init)..." - incus exec "$instance" -- cloud-init status --wait + # A failed cloud-init used to print a screen of dots and the word "error", + # with nothing to act on — the box's own log holds the reason, and nobody + # was told it existed. Show it, and leave the box up to inspect. + if ! incus exec "$instance" -- cloud-init status --wait; then + echo >&2 + echo "claudebox: cloud-init FAILED in $name. What it says:" >&2 + incus exec "$instance" -- cloud-init status --long 2>&1 | sed 's/^/ /' >&2 + echo >&2 + echo "claudebox: the errors, from the box's log:" >&2 + incus exec "$instance" -- sh -c \ + "grep -iE '^(E:|Err:)|Temporary failure|Could not resolve|Unable to fetch' /var/log/cloud-init-output.log | tail -8" \ + 2>/dev/null | sed 's/^/ /' >&2 + echo >&2 + echo "claudebox: '$name' is still up — inspect it, then delete it:" >&2 + echo " claudebox incus $name -- exec {} -- tail -50 /var/log/cloud-init-output.log" >&2 + echo " claudebox rm $name" >&2 + echo "claudebox: a box that cannot resolve DNS is usually the HOST's resolver" >&2 + echo " (a VPN/Tailscale resolver the box inherits — see issue #33)." >&2 + die "cloud-init failed — the box is incomplete, so refusing to hand it over" + fi fi echo "claudebox: ready — 'claudebox shell $name'. Log into Claude inside: run 'claude' then /login." } diff --git a/drill/doctor.sh b/drill/doctor.sh index dc4ec9d..ac26f9b 100755 --- a/drill/doctor.sh +++ b/drill/doctor.sh @@ -15,8 +15,13 @@ # question that kept getting answered by hand. set -u -FIX=0 -[ "${1:-}" = "--fix" ] && FIX=1 +FIX=0; PIN=0 +case "${1:-}" in + --fix) FIX=1 ;; + --pin-dns) PIN=1 ;; + "") ;; + *) echo "usage: doctor.sh [--fix | --pin-dns]"; exit 2 ;; +esac bad=0 ok() { printf ' \033[32mOK\033[0m %s\n' "$*"; } @@ -90,22 +95,65 @@ for b in drill clone archive peer payroll cbprobe cbcopy cbnotours; do fi done +# --- the box's DNS comes from the HOST's resolver. See issue #33. ----------- +head_ "Host resolver — a box's DNS is forwarded through this" +hostns="$(grep -E '^nameserver' /etc/resolv.conf 2>/dev/null | awk '{print $2}' | tr '\n' ' ')" +inf "/etc/resolv.conf: ${hostns:-}" +raw="$(incus network get claudenet raw.dnsmasq 2>/dev/null | tr '\n' ';')" +if [ -n "$raw" ]; then + ok "claudenet has a pinned resolver (raw.dnsmasq: $raw)" + inf "boxes do NOT inherit the host's resolver — good (issue #33)" +else + # 100.64.0.0/10 is CGNAT — which is exactly Tailscale's range. + if printf '%s' "$hostns" | grep -qE '(^| )100\.(6[4-9]|[7-9][0-9]|1[01][0-9]|12[0-7])\.'; then + no "the host resolves via a CGNAT/Tailscale resolver ($hostns), and boxes INHERIT it — see issue #33" + inf "· box DNS breaks whenever the tailnet's resolver does (this is what kills cold mints)" + inf "· and tailnet names RESOLVE from inside a box, though its ACL blocks connecting to them" + inf "fix + test: bash drill/doctor.sh --pin-dns" + else + inf "boxes inherit the host's resolver (unpinned). Fine while it is stable; see issue #33." + fi +fi + +if [ "$PIN" = 1 ]; then + head_ "Pinning claudenet's resolver (issue #33)" + if incus network set claudenet raw.dnsmasq "$(printf 'no-resolv\nserver=1.1.1.1\nserver=8.8.8.8\n')" 2>/tmp/pin.err; then + ok "set raw.dnsmasq: no-resolv + 1.1.1.1 + 8.8.8.8 — dnsmasq now ignores /etc/resolv.conf" + inf "a box's DNS no longer depends on the host's VPN state, and MagicDNS is out of the path" + inf "re-run the drill; if the cold mint now succeeds, issue #33 is confirmed and the fix belongs in setup-host.sh" + else + no "incus rejected raw.dnsmasq: $(head -1 /tmp/pin.err 2>/dev/null)" + inf "then raw.dnsmasq is the wrong lever and #33 needs a different mechanism — say so on the issue" + fi +fi + head_ "Can a box actually resolve DNS?" probe="" for b in drill archive peer clone; do incus config show "$b" >/dev/null 2>&1 && { probe="$b"; break; } done if [ -n "$probe" ] && [ "$FIX" != 1 ]; then - inf "probing inside '$probe' (the cheapest test of a poisoned network):" - inf "resolv.conf: $(timeout 20 incus exec "$probe" -- sh -c 'grep -m2 nameserver /etc/resolv.conf' 2>/dev/null | tr '\n' ' ')" - if timeout 25 incus exec "$probe" -- getent hosts deb.debian.org >/dev/null 2>&1; then - ok "$probe resolves deb.debian.org" - else - no "$probe CANNOT resolve deb.debian.org — this is what breaks cloud-init on every new box" - fi + inf "probing inside '$probe' — this separates DNS from routing, which is the whole question:" + inf "its resolv.conf: $(timeout 20 incus exec "$probe" -- sh -c 'grep -m2 nameserver /etc/resolv.conf' 2>/dev/null | tr '\n' ' ')" + timeout 20 incus exec "$probe" -- ping -c1 -W2 10.87.0.1 >/dev/null 2>&1 \ - && ok "$probe reaches the gateway (10.87.0.1) — so it is DNS, not routing" \ - || no "$probe cannot even reach the gateway" + && ok "reaches the gateway (10.87.0.1) — routing is fine" \ + || no "cannot even reach the gateway — this is routing, not DNS" + + if timeout 25 incus exec "$probe" -- getent hosts deb.debian.org >/dev/null 2>&1; then + ok "resolves deb.debian.org — DNS works" + else + no "CANNOT resolve deb.debian.org — this is exactly what kills cloud-init on every cold mint" + # Does the box reach the internet at all WITHOUT DNS? If yes, the fault is + # purely name resolution — i.e. the forwarder, i.e. issue #33. + if timeout 25 incus exec "$probe" -- curl -sS -m 10 -o /dev/null https://1.1.1.1 2>/dev/null; then + inf "…but it CAN reach 1.1.1.1 by address. So egress works and only NAME RESOLUTION is broken:" + inf "the fault is the forwarder the box inherits from the host — issue #33." + inf "test the fix: bash drill/doctor.sh --pin-dns then re-run the drill" + else + inf "…and it cannot reach 1.1.1.1 by address either — so egress itself is broken, not just DNS." + fi + fi else inf "no box to probe with (mint one, or run without --fix after a run)" fi -- 2.45.2