claude-hdb
28be7a0ec7
Two cold mints in a row died with cloud-init 'status: error' on a host
the doctor had just certified clean — so the earlier "leftover mutations
poisoned the network" theory is dead, and the DNS failure is
reproducible rather than transient.
'claudebox new' printed four hundred dots and the word "error", leaving
the user with nothing to act on: the reason was in the box's own log and
nobody was told the log existed. It now prints cloud-init's status, the
fetch/resolve errors from the box's log, and how to inspect the box —
which is left running, because a box that failed to build is evidence,
not garbage. It also names the usual culprit: the host's resolver.
doctor.sh gains the diagnosis that keeps being done by hand:
· what the HOST resolves through, and whether that is a CGNAT/Tailscale
resolver the boxes inherit (issue #33);
· whether claudenet's resolver is pinned;
· and inside a box, the question that settles it — DNS is broken, but
can it still reach 1.1.1.1 BY ADDRESS? If yes, egress is fine and the
fault is purely the inherited forwarder.
· --pin-dns applies the #33 fix (raw.dnsmasq: no-resolv + public
servers) so the hypothesis can be TESTED rather than argued.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
172 lines
8 KiB
Bash
Executable file
172 lines
8 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# doctor.sh — is this host in a fit state to drill, and if not, what is wrong?
|
|
#
|
|
# bash drill/doctor.sh # report
|
|
# bash drill/doctor.sh --fix # report, then revert what the drill left behind
|
|
#
|
|
# The drill MUTATES the host in phase D (dns.mode, NIC filtering, ACL rules) to
|
|
# rehearse the #16 hardening. If a run aborts before it reverts them, those
|
|
# mutations outlive it — and the next run mints boxes on a broken network. That
|
|
# is not a hypothetical: it is how a box came up with no DNS at all
|
|
# ("Temporary failure resolving deb.debian.org" in cloud-init), and how a false
|
|
# design veto against #16 got posted from a poisoned baseline.
|
|
#
|
|
# This script is the answer to "what state is the host actually in?" — the
|
|
# question that kept getting answered by hand.
|
|
set -u
|
|
|
|
FIX=0; PIN=0
|
|
case "${1:-}" in
|
|
--fix) FIX=1 ;;
|
|
--pin-dns) PIN=1 ;;
|
|
"") ;;
|
|
*) echo "usage: doctor.sh [--fix | --pin-dns]"; exit 2 ;;
|
|
esac
|
|
|
|
bad=0
|
|
ok() { printf ' \033[32mOK\033[0m %s\n' "$*"; }
|
|
no() { printf ' \033[31mDIRTY\033[0m %s\n' "$*"; bad=$((bad + 1)); }
|
|
inf() { printf ' %s\n' "$*"; }
|
|
head_() { printf '\n\033[1m%s\033[0m\n' "$*"; }
|
|
|
|
command -v incus >/dev/null || { echo "doctor: incus is not installed on this host."; exit 1; }
|
|
timeout 10 incus list >/dev/null 2>&1 || {
|
|
echo "doctor: the incus daemon is not answering (see issue #26 for recovery):"
|
|
echo " sudo pkill -9 -f 'incusd shutdown'"
|
|
echo " sudo systemctl stop incus.service incus.socket"
|
|
echo " sudo systemctl reset-failed incus.service incus.socket"
|
|
echo " sudo systemctl start incus.socket incus.service"
|
|
exit 1
|
|
}
|
|
|
|
head_ "Network — claudenet"
|
|
if incus network show claudenet >/dev/null 2>&1; then
|
|
dns="$(incus network get claudenet dns.mode 2>/dev/null)"
|
|
if [ -z "$dns" ] || [ "$dns" = managed ]; then
|
|
ok "dns.mode = ${dns:-<unset, i.e. managed>}"
|
|
else
|
|
no "dns.mode = $dns ← the drill's phase D left this behind. Boxes minted now get NO working DNS."
|
|
[ "$FIX" = 1 ] && { incus network unset claudenet dns.mode && inf "reverted: dns.mode unset"; }
|
|
fi
|
|
inf "ipv4.address = $(incus network get claudenet ipv4.address 2>/dev/null)"
|
|
ipv6="$(incus network get claudenet ipv6.address 2>/dev/null)"
|
|
[ "$ipv6" = none ] && ok "ipv6.address = none (the isolation contract — every ACL rule is IPv4-only)" \
|
|
|| no "ipv6.address = $ipv6 — IPv6 is on and NOT covered by any ACL rule"
|
|
else
|
|
inf "claudenet does not exist (a fresh host — setup-host.sh will create it)"
|
|
fi
|
|
|
|
head_ "Profile — claude-dev (the NIC is the isolation contract)"
|
|
if incus profile show claude-dev >/dev/null 2>&1; then
|
|
for k in security.mac_filtering security.ipv4_filtering; do
|
|
v="$(incus profile device get claude-dev eth0 "$k" 2>/dev/null)"
|
|
if [ -z "$v" ]; then
|
|
ok "$k unset (as shipped)"
|
|
else
|
|
no "$k = $v ← phase D left this behind. A box can fail to get on the network at all."
|
|
[ "$FIX" = 1 ] && { incus profile device unset claude-dev eth0 "$k" && inf "reverted: $k unset"; }
|
|
fi
|
|
done
|
|
inf "cpu/mem: $(incus profile get claude-dev limits.cpu 2>/dev/null)/$(incus profile get claude-dev limits.memory 2>/dev/null) (the drill lowers these on a small host)"
|
|
else
|
|
inf "claude-dev does not exist (a fresh host)"
|
|
fi
|
|
|
|
head_ "ACL — claude-isolate"
|
|
if incus network acl show claude-isolate >/dev/null 2>&1; then
|
|
n="$(incus network acl show claude-isolate | grep -c 'action:' || true)"
|
|
inf "$n rules"
|
|
incus network acl show claude-isolate | grep -E 'action:|destination:' | sed 's/^/ /'
|
|
if incus network acl show claude-isolate | grep -q '@internal'; then
|
|
no "an @internal rule survived phase D"
|
|
[ "$FIX" = 1 ] && { incus network acl rule remove claude-isolate egress action=drop destination=@internal && inf "reverted: @internal rule removed"; }
|
|
fi
|
|
else
|
|
inf "claude-isolate does not exist (a fresh host)"
|
|
fi
|
|
|
|
head_ "Instances"
|
|
left="$(incus list --format csv --columns ns 2>/dev/null)"
|
|
[ -z "$left" ] && inf "(none)" || printf ' %s\n' "$left"
|
|
for b in drill clone archive peer payroll cbprobe cbcopy cbnotours; do
|
|
if incus config show "$b" >/dev/null 2>&1; then
|
|
no "leftover drill box: $b"
|
|
[ "$FIX" = 1 ] && { timeout 60 incus delete -f "$b" >/dev/null 2>&1 && inf "reverted: deleted $b"; }
|
|
fi
|
|
done
|
|
|
|
# --- the box's DNS comes from the HOST's resolver. See issue #33. -----------
|
|
head_ "Host resolver — a box's DNS is forwarded through this"
|
|
hostns="$(grep -E '^nameserver' /etc/resolv.conf 2>/dev/null | awk '{print $2}' | tr '\n' ' ')"
|
|
inf "/etc/resolv.conf: ${hostns:-<none>}"
|
|
raw="$(incus network get claudenet raw.dnsmasq 2>/dev/null | tr '\n' ';')"
|
|
if [ -n "$raw" ]; then
|
|
ok "claudenet has a pinned resolver (raw.dnsmasq: $raw)"
|
|
inf "boxes do NOT inherit the host's resolver — good (issue #33)"
|
|
else
|
|
# 100.64.0.0/10 is CGNAT — which is exactly Tailscale's range.
|
|
if printf '%s' "$hostns" | grep -qE '(^| )100\.(6[4-9]|[7-9][0-9]|1[01][0-9]|12[0-7])\.'; then
|
|
no "the host resolves via a CGNAT/Tailscale resolver ($hostns), and boxes INHERIT it — see issue #33"
|
|
inf "· box DNS breaks whenever the tailnet's resolver does (this is what kills cold mints)"
|
|
inf "· and tailnet names RESOLVE from inside a box, though its ACL blocks connecting to them"
|
|
inf "fix + test: bash drill/doctor.sh --pin-dns"
|
|
else
|
|
inf "boxes inherit the host's resolver (unpinned). Fine while it is stable; see issue #33."
|
|
fi
|
|
fi
|
|
|
|
if [ "$PIN" = 1 ]; then
|
|
head_ "Pinning claudenet's resolver (issue #33)"
|
|
if incus network set claudenet raw.dnsmasq "$(printf 'no-resolv\nserver=1.1.1.1\nserver=8.8.8.8\n')" 2>/tmp/pin.err; then
|
|
ok "set raw.dnsmasq: no-resolv + 1.1.1.1 + 8.8.8.8 — dnsmasq now ignores /etc/resolv.conf"
|
|
inf "a box's DNS no longer depends on the host's VPN state, and MagicDNS is out of the path"
|
|
inf "re-run the drill; if the cold mint now succeeds, issue #33 is confirmed and the fix belongs in setup-host.sh"
|
|
else
|
|
no "incus rejected raw.dnsmasq: $(head -1 /tmp/pin.err 2>/dev/null)"
|
|
inf "then raw.dnsmasq is the wrong lever and #33 needs a different mechanism — say so on the issue"
|
|
fi
|
|
fi
|
|
|
|
head_ "Can a box actually resolve DNS?"
|
|
probe=""
|
|
for b in drill archive peer clone; do
|
|
incus config show "$b" >/dev/null 2>&1 && { probe="$b"; break; }
|
|
done
|
|
if [ -n "$probe" ] && [ "$FIX" != 1 ]; then
|
|
inf "probing inside '$probe' — this separates DNS from routing, which is the whole question:"
|
|
inf "its resolv.conf: $(timeout 20 incus exec "$probe" -- sh -c 'grep -m2 nameserver /etc/resolv.conf' 2>/dev/null | tr '\n' ' ')"
|
|
|
|
timeout 20 incus exec "$probe" -- ping -c1 -W2 10.87.0.1 >/dev/null 2>&1 \
|
|
&& ok "reaches the gateway (10.87.0.1) — routing is fine" \
|
|
|| no "cannot even reach the gateway — this is routing, not DNS"
|
|
|
|
if timeout 25 incus exec "$probe" -- getent hosts deb.debian.org >/dev/null 2>&1; then
|
|
ok "resolves deb.debian.org — DNS works"
|
|
else
|
|
no "CANNOT resolve deb.debian.org — this is exactly what kills cloud-init on every cold mint"
|
|
# Does the box reach the internet at all WITHOUT DNS? If yes, the fault is
|
|
# purely name resolution — i.e. the forwarder, i.e. issue #33.
|
|
if timeout 25 incus exec "$probe" -- curl -sS -m 10 -o /dev/null https://1.1.1.1 2>/dev/null; then
|
|
inf "…but it CAN reach 1.1.1.1 by address. So egress works and only NAME RESOLUTION is broken:"
|
|
inf "the fault is the forwarder the box inherits from the host — issue #33."
|
|
inf "test the fix: bash drill/doctor.sh --pin-dns then re-run the drill"
|
|
else
|
|
inf "…and it cannot reach 1.1.1.1 by address either — so egress itself is broken, not just DNS."
|
|
fi
|
|
fi
|
|
else
|
|
inf "no box to probe with (mint one, or run without --fix after a run)"
|
|
fi
|
|
|
|
head_ "Verdict"
|
|
if [ "$bad" -eq 0 ]; then
|
|
printf ' \033[32mclean\033[0m — this host is fit to drill.\n\n'
|
|
exit 0
|
|
fi
|
|
printf ' \033[31m%s problem(s)\033[0m — this host is NOT fit to drill.\n' "$bad"
|
|
if [ "$FIX" = 1 ]; then
|
|
printf ' reverted what could be reverted; re-run doctor to confirm.\n\n'
|
|
else
|
|
printf ' run: bash drill/doctor.sh --fix\n\n'
|
|
fi
|
|
exit 1
|