From d16f230e3b6ff1d60bcc9e27d3c7aaa2855bce4f Mon Sep 17 00:00:00 2001 From: dan-claude-bot Date: Sat, 18 Jul 2026 19:47:20 +0000 Subject: [PATCH] feat(doctor): recognize the #80 gateway-squat signature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The doctor had already CAUGHT #80 without knowing it: 'DIRTY cannot reach 1.1.1.1 by address' beside 'OK resolves deb.debian.org' — and attributed it to the host admin, whose every check was green. That split IS the fingerprint: the impostor dnsmasq on the captured gateway address keeps answering names (it forwards upstream via the default route) while direct IP egress dies. Three additions: · gw_squat_signature — pure text (ip -4 route show / ip -4 -o addr show) in, findings out: a default gateway held as a LOCAL address (the kernel's local table then eats packets meant for the real gateway — DNS, unicast DHCP renewals), and duplicate connected routes for the uplink subnet (whichever link last gains carrier wins, and a nested bridge with carrier blackholes egress). Pure so test/cli.sh drives it against synthetic tables, including the issue's exact captures. · judged on THIS machine first, both tiers, before any daemon call — inside a poisoned guest the daemon answering is the nested impostor, happily judging its own stack clean — and probed INSIDE every box the doctor examines (probe_sig), where the signature actually lives. · the admin ACL section now compares the gateway carve-out to boxnet's actual gateway: the #80 escape hatch moves the bridge, and the stale /32 strands box DNS inside the 10.0.0.0/8 drop. Measured live on a workaround host: doctor names the stale carve-out and the BOX_SUBNET re-run that converges it. The egress-broken-but-DNS-fine hint now names itself on both tiers and points at the signature probes for the verdict. Co-Authored-By: Claude Fable 5 --- drill/doctor.sh | 112 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 106 insertions(+), 6 deletions(-) diff --git a/drill/doctor.sh b/drill/doctor.sh index f8de671..4fac72b 100755 --- a/drill/doctor.sh +++ b/drill/doctor.sh @@ -35,7 +35,75 @@ no() { printf ' \033[31mDIRTY\033[0m %s\n' "$*"; bad=$((bad + 1)); } inf() { printf ' %s\n' "$*"; } head_() { printf '\n\033[1m%s\033[0m\n' "$*"; } +# --- The #80 signature: a nested box stack squatting on the gateway --------- +# setup-host run INSIDE a box builds a nested boxnet on the guest's own uplink +# subnet. The measured mechanism, and the two lines this function reads for: +# hold your own gateway's address and the kernel's local table eats packets +# meant for the real gateway (DNS, unicast DHCP renewals); carry two connected +# routes for the uplink subnet and whichever link last has carrier wins — a +# nested bridge gaining carrier blackholes egress instantly. Pure text in, +# findings out (one per line, silence is clean), so test/cli.sh drives it +# against synthetic route tables and the guest probe can feed it routes read +# INSIDE a box. Inputs: `ip -4 route show` and `ip -4 -o addr show` output. +gw_squat_signature() { + local routes="$1" addrs="$2" gw updev + gw="$(printf '%s\n' "$routes" | awk '$1 == "default" { for (i = 1; i < NF; i++) if ($i == "via") { print $(i+1); exit } }')" + updev="$(printf '%s\n' "$routes" | awk '$1 == "default" { for (i = 1; i < NF; i++) if ($i == "dev") { print $(i+1); exit } }')" + [ -n "$gw" ] || return 0 # no default route: nothing to squat on + printf '%s\n' "$addrs" | awk -v gw="$gw" ' + { split($4, a, "/") + if (a[1] == gw) { print "the default gateway " gw " is held as a LOCAL address (on " $2 ") — the kernel delivers packets meant for the gateway to this machine itself"; exit } }' + printf '%s\n' "$routes" | awk -v updev="$updev" ' + / proto kernel / && $2 == "dev" { + cnt[$1]++; devs[$1] = devs[$1] (devs[$1] ? ", " : "") $3 + if ($3 == updev) up = $1 + } + END { if (up != "" && cnt[up] > 1) + print "duplicate connected routes for the uplink subnet " up " (" devs[up] ") — whichever link last gains carrier wins, and a nested bridge with carrier blackholes egress" }' +} + +# The signature, probed INSIDE a box: its routes, read where they live. A +# poisoned guest looks healthy from every host-side config check — the nested +# bridge and the captured gateway exist only in the guest's kernel. +probe_sig() { + local b="$1" routes addrs sig line + routes="$(timeout -k 5 20 incus exec "$b" -- ip -4 route show /dev/null)" + addrs="$(timeout -k 5 20 incus exec "$b" -- ip -4 -o addr show /dev/null)" + if [ -z "$routes" ]; then + inf "could not read routes inside '$b' — the #80 signature was not probed" + return 0 + fi + sig="$(gw_squat_signature "$routes" "$addrs")" + if [ -n "$sig" ]; then + while IFS= read -r line; do no "inside '$b': $line"; done <<<"$sig" + inf "a box stack was installed INSIDE this box — its nested bridge claims the" + inf "box's own uplink subnet, and egress blacks out intermittently (issue #80)." + inf "fix, inside the box: sudo incus network set boxnet ipv4.address 10.89.0.1/24" + inf " (or remove the nested stack there: box teardown-host)" + else + ok "no #80 signature inside '$b' — nothing is squatting on its gateway" + fi +} + command -v incus >/dev/null || { echo "doctor: incus is not installed on this host."; exit 1; } + +# THIS MACHINE first, both tiers, before anything that needs the daemon: the +# #80 signature is a fact about the kernel's routing tables, not about incus — +# and a poisoned guest is exactly where the daemon answering below may be the +# WRONG (nested) one, judging its own impostor stack clean. +head_ "This machine — is a nested box stack squatting on the gateway? (#80)" +sig="$(gw_squat_signature "$(ip -4 route show 2>/dev/null)" "$(ip -4 -o addr show 2>/dev/null)")" +if [ -n "$sig" ]; then + while IFS= read -r line; do no "$line"; done <<<"$sig" + inf "a box stack was built on a machine whose uplink already owns its subnet —" + inf "run inside a box, that is issue #80: egress blacks out intermittently while" + inf "everything looks healthy. setup-host now refuses this; this machine already has it." + inf "fix: move the nested bridge off the uplink's subnet:" + inf " sudo incus network set boxnet ipv4.address 10.89.0.1/24" + inf " (or remove the nested stack: box teardown-host)" +else + ok "the default gateway is not held locally, and the uplink subnet has one connected route" +fi timeout 10 incus list >/dev/null 2>&1 || { echo "doctor: the incus daemon is not answering (see issue #26 for recovery):" echo " sudo pkill -9 -f 'incusd shutdown'" @@ -85,12 +153,21 @@ if [ "$TIER" = restricted ]; then | awk -F, '$2 == "RUNNING" { print $1; exit }')" if [ -n "$probe" ]; then inf "probing inside '$probe':" - timeout -k 5 25 incus exec "$probe" -- curl -sS -m 10 -o /dev/null https://1.1.1.1 /dev/null \ - && ok "reaches 1.1.1.1 by address — egress routing is fine" \ - || no "cannot reach 1.1.1.1 by address — egress routing is broken (an admin problem: box doctor as admin)" - timeout -k 5 25 incus exec "$probe" -- getent hosts deb.debian.org /dev/null 2>&1 \ - && ok "resolves deb.debian.org — DNS works" \ - || no "CANNOT resolve deb.debian.org — an admin problem (the resolver pin lives on the host): box doctor as admin" + if timeout -k 5 25 incus exec "$probe" -- curl -sS -m 10 -o /dev/null https://1.1.1.1 /dev/null; then + routing=1; ok "reaches 1.1.1.1 by address — egress routing is fine" + else + routing=0; no "cannot reach 1.1.1.1 by address — egress routing is broken (an admin problem: box doctor as admin)" + fi + if timeout -k 5 25 incus exec "$probe" -- getent hosts deb.debian.org /dev/null 2>&1; then + ok "resolves deb.debian.org — DNS works" + # Egress broken while DNS resolves is #80's fingerprint: an impostor + # dnsmasq on a captured gateway address answers names happily (it + # forwards upstream via the default route) while direct IP egress dies. + [ "$routing" = 0 ] && inf "…egress broken while DNS resolves is #80's fingerprint — the signature probe below answers whether something inside this box squats on its gateway" + else + no "CANNOT resolve deb.debian.org — an admin problem (the resolver pin lives on the host): box doctor as admin" + fi + probe_sig "$probe" else inf "no running box to probe with (mint one: box new --name work)" fi @@ -209,6 +286,22 @@ if incus network acl show box-isolate >/dev/null 2>&1; then no "an @internal rule survived phase D" [ "$FIX" = 1 ] && { incus network acl rule remove box-isolate egress action=drop destination=@internal && inf "reverted: @internal rule removed"; } fi + # The gateway carve-out must track the BRIDGE. #80's escape hatch moves + # boxnet off a colliding subnet — and the stale /32 then strands box DNS + # inside the 10.0.0.0/8 drop, which presents as a dead resolver, never as + # a stale ACL. Compare the allow rule to boxnet's actual gateway. + gwaddr="$(incus network get boxnet ipv4.address 2>/dev/null | cut -d/ -f1)" + carve="$(incus network acl show box-isolate 2>/dev/null \ + | awk '/- action: allow/ { hit = 1; next } hit && /destination:/ { sub("/32", "", $2); print $2; exit } { hit = 0 }')" + if [ -n "$gwaddr" ] && [ -n "$carve" ]; then + if [ "$carve" = "$gwaddr" ]; then + ok "the gateway carve-out matches boxnet's gateway ($gwaddr) — box DNS survives the 10/8 drop" + else + no "the gateway carve-out ($carve/32) does NOT match boxnet's gateway ($gwaddr) — box DNS to the gateway dies inside the 10.0.0.0/8 drop" + inf "the bridge moved (#80's escape hatch) and the ACL did not follow" + inf "fix: BOX_SUBNET=${gwaddr%.*}.0/24 box setup-host (it converges the ACL now)" + fi + fi else inf "box-isolate does not exist (a fresh host)" fi @@ -329,6 +422,11 @@ if [ -n "$probe" ] && [ "$FIX" != 1 ]; then if timeout -k 5 25 incus exec "$probe" -- getent hosts deb.debian.org /dev/null 2>&1; then ok "resolves deb.debian.org — DNS works" + # The OTHER split from the one below: egress broken while DNS resolves is + # #80's fingerprint — an impostor dnsmasq on a captured gateway address + # keeps answering names (it forwards upstream via the default route) + # while direct IP egress dies. The signature probe underneath answers it. + [ "$routing" = 0 ] && inf "…egress broken while DNS resolves is #80's fingerprint — see the signature probe below" else no "CANNOT resolve deb.debian.org — this is exactly what kills cloud-init on every cold mint" # Egress by address was probed above. If it worked, the fault is purely @@ -341,6 +439,8 @@ if [ -n "$probe" ] && [ "$FIX" != 1 ]; then inf "…and it cannot reach 1.1.1.1 by address either — so egress itself is broken, not just DNS." fi fi + + probe_sig "$probe" else inf "no box to probe with (mint one, or run without --fix after a run)" fi