From b429f5b65c99e70db74b7ca12a6f0392728465a9 Mon Sep 17 00:00:00 2001 From: dan-claude-bot Date: Sat, 18 Jul 2026 19:46:58 +0000 Subject: [PATCH 1/4] feat(setup-host): refuse a claimed subnet, and BOX_SUBNET to move off one (#80) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run inside a box, setup-host built a nested boxnet claiming the exact subnet and gateway of the guest's own uplink: the guest then held 10.88.0.1 as a LOCAL address while its default route pointed at 10.88.0.1 as its GATEWAY, carried two connected routes for the subnet, and suffered intermittent, self-recovering egress blackouts (~24-36 s, roughly hourly) that no host-side check could attribute. The flagship use case funnels agents toward exactly this — working on box, in a box — so the guard, not the operator, has to catch it. setup-host now scans the target subnet BEFORE any mutation: the default route's gateway inside it (the smoking gun — that is this machine's own uplink), or any non-boxnet interface holding an address in it, refuses with the way out named (BOX_SUBNET). A prior boxnet owning the subnet is the one claimant that is NOT a collision — the legitimate re-run — and an existing bridge on a DIFFERENT subnet refuses too: setup-host converges a bridge, it never re-addresses one under live leases. BOX_SUBNET= (validated at the gate, alongside BOX_DNS) moves the whole stack from one place: · the bridge address derives (ipv4.address=/24) · the ACL's gateway carve-out derives — and the ruleset is now converged via 'network acl edit', not created once: the #80 escape hatch (moving the bridge) left the old /32 behind, stranding box DNS inside the 10.0.0.0/8 drop, which presents as a dead resolver, never as a stale ACL · box-firewall reads the gateway off the live bridge ('|| true' because under pipefail an absent bridge would kill the script instead of answering "no bridge yet") · the drill, multiuser rehearsal and migrate-host probes derive the prefix from 'incus network get boxnet ipv4.address' instead of hardcoding 10.88 — a BOX_SUBNET host must not fail its own rehearsals Co-Authored-By: Claude Fable 5 --- drill/drill.sh | 22 +++++--- drill/multiuser.sh | 13 +++-- host/box-firewall.sh | 10 +++- host/migrate-host.sh | 14 +++-- host/setup-host.sh | 131 ++++++++++++++++++++++++++++++++++++++----- 5 files changed, 157 insertions(+), 33 deletions(-) diff --git a/drill/drill.sh b/drill/drill.sh index 257e715..c40d8be 100755 --- a/drill/drill.sh +++ b/drill/drill.sh @@ -94,14 +94,17 @@ in_box() { # · and the interface is NOT called eth0. The PROFILE names the device eth0, # but inside a VM guest predictable naming renames it enp5s0. Six runs of # A3 "not probed" were this, not the network. -# So: read it from inside the box, and select by SUBNET (10.88.x, what boxnet -# hands out) rather than by interface name — docker0 (172.17.x) is the decoy, -# and the NIC's name is the guest's business, not ours. +# So: read it from inside the box, and select by SUBNET (what boxnet hands +# out — read off the network, never hardcoded: BOX_SUBNET moves it, #80) +# rather than by interface name — docker0 (172.17.x) is the decoy, and the +# NIC's name is the guest's business, not ours. +boxnet_gw() { incus network get boxnet ipv4.address 2>/dev/null | cut -d/ -f1; } boxnet_ip() { - local b="$1" ip _i + local b="$1" ip _i pfx + pfx="$(boxnet_gw)"; pfx="${pfx%.*}." for _i in $(seq 1 15); do ip="$(in_box "$b" ip -4 -o addr show scope global \ - | awk '{ for (i = 1; i < NF; i++) if ($i == "inet" && $(i+1) ~ /^10\.88\./) { split($(i+1), a, "/"); print a[1]; exit } }')" + | awk -v p="$pfx" '{ for (i = 1; i < NF; i++) if ($i == "inet" && index($(i+1), p) == 1) { split($(i+1), a, "/"); print a[1]; exit } }')" [ -n "$ip" ] && { printf '%s\n' "$ip"; return 0; } sleep 2 done @@ -693,10 +696,11 @@ fi # nothing serves and read refused-vs-dropped — refused would mean the box's # packet reached the host's stack, which is the thing the firewall must prevent. # (No background listener: one less process to leak, one less way to wedge.) -hv="$(box_probe archive http://10.88.0.1:8099)" +gw="$(boxnet_gw)" +hv="$(box_probe archive "http://$gw:8099")" case "$hv" in reachable|refused) - no "THE BOX'S PACKETS REACH THE HOST on 10.88.0.1:8099 [$hv] — the firewall rules are not holding" + no "THE BOX'S PACKETS REACH THE HOST on $gw:8099 [$hv] — the firewall rules are not holding" aud "A2 box→host: FAIL — $hv (the packet reached the host's stack)" ;; dropped) ok "box → host is blocked (no path to the machine's sockets)" @@ -747,7 +751,7 @@ elif [ -n "$PEER_IP" ]; then fi else no "could not read peer's boxnet address — the sibling probe never ran" - aud "A3 sibling: NOT PROBED (no 10.88.x address on peer)" + aud "A3 sibling: NOT PROBED (no boxnet address on peer)" fi # C5 — DNS enumeration (#15 A4). Now a CONTRACT, not an observation: setup-host @@ -936,7 +940,7 @@ else || no "migrate: legacy box is NOT on box-net" lip="$(boxnet_ip legacybox)" [ -n "$lip" ] && ok "migrate: legacy box got a boxnet address ($lip) — network move landed" \ - || no "migrate: legacy box has no 10.88 address — the move did not take" + || no "migrate: legacy box has no boxnet address — the move did not take" in_box legacybox getent hosts deb.debian.org >/dev/null 2>&1 \ && ok "migrate: re-homed box resolves + reaches the internet on its new leg" \ || no "migrate: re-homed box cannot resolve on boxnet" diff --git a/drill/multiuser.sh b/drill/multiuser.sh index a49fe5d..9de9616 100644 --- a/drill/multiuser.sh +++ b/drill/multiuser.sh @@ -105,6 +105,11 @@ probe_up() { # probe_up echo "$r" } +# The hardened network's gateway and prefix, read off the network — never +# hardcoded, because BOX_SUBNET moves the whole subnet now (#80). +boxnet_gw() { incus network get boxnet ipv4.address 2>/dev/null | cut -d/ -f1; } +boxnet_pfx() { local gw; gw="$(boxnet_gw)"; printf '%s.' "${gw%.*}"; } + cleanup() { [ "$KEEP" = 1 ] && { echo "(--keep: users and boxes left for inspection)"; return; } echo @@ -194,8 +199,8 @@ fi rm -f "$mintlog" as_u "$U1" box list 2>/dev/null | grep -q '^mine ' && ok "(b) box list shows mine" || no "(b) box list does not show mine" as_u "$U1" box exec mine -- true >/dev/null 2>&1 && ok "(b) box exec mine -- true" || no "(b) box exec failed" -as_u "$U1" box info mine 2>/dev/null | grep -q '10\.88\.' \ - && ok "(g) box info shows a boxnet (10.88.x) address — placed on the hardened network" \ +as_u "$U1" box info mine 2>/dev/null | grep -qF "$(boxnet_pfx)" \ + && ok "(g) box info shows a boxnet ($(boxnet_pfx)x) address — placed on the hardened network" \ || no "(g) mine has no boxnet address in box info" as_u "$U1" box snapshot mine s1 >/dev/null 2>&1 && ok "(b) box snapshot mine s1" || no "(b) snapshot refused" as_u "$U1" box restore mine s1 >/dev/null 2>&1 && ok "(b) box restore mine s1 (the incus 6 'snapshot restore' spelling)" || no "(b) restore failed" @@ -224,14 +229,14 @@ phase "g. the isolation contract, measured from INSIDE the boxes" ip1="$(incus --project "$p1" list mine --format csv --columns 4 2>/dev/null | tr -d '"' | sed 's/ (.*//' | head -n1)" ip2="$(incus --project "$p2" list mine --format csv --columns 4 2>/dev/null | tr -d '"' | sed 's/ (.*//' | head -n1)" inf "$U1's mine: ${ip1:-} $U2's mine: ${ip2:-}" -case "$ip1" in 10.88.*) ok "(g) $U1's box holds a boxnet lease" ;; *) no "(g) $U1's box is NOT on boxnet: '$ip1'" ;; esac +case "$ip1" in "$(boxnet_pfx)"*) ok "(g) $U1's box holds a boxnet lease" ;; *) no "(g) $U1's box is NOT on boxnet: '$ip1'" ;; esac r="$(probe_up "$U1" mine https://1.1.1.1)" [ "$r" = reachable ] && ok "(g) egress to the public internet works (curl 1.1.1.1: $r)" || no "(g) public egress broken: $r" as_u "$U1" timeout -k 5 20 incus exec mine -- getent hosts deb.debian.org >/dev/null 2>&1 \ && ok "(g) public DNS resolves (via the pinned resolver)" || no "(g) DNS broken inside the box" -r="$(probe_from "$U1" mine "http://10.88.0.1:22")" +r="$(probe_from "$U1" mine "http://$(boxnet_gw):22")" [ "$r" = dropped ] && ok "(g) box → host is dropped (gateway :22: $r)" || no "(g) box can reach the HOST: $r" r="$(probe_from "$U1" mine "http://192.168.0.1")" [ "$r" = dropped ] && ok "(g) box → RFC1918 is dropped ($r)" || no "(g) box reaches private space: $r" diff --git a/host/box-firewall.sh b/host/box-firewall.sh index edd8374..bb96b50 100644 --- a/host/box-firewall.sh +++ b/host/box-firewall.sh @@ -5,8 +5,16 @@ # Docker's DOCKER-USER rules are runtime-only and need re-applying). set -euo pipefail -GW=10.88.0.1 NET=boxnet +# The gateway is read off the live bridge, not hardcoded: the subnet is an +# input now (BOX_SUBNET, setup-host.sh — #80), and a bridge moved off a +# colliding subnet must keep its firewall. setup-host runs us after the +# bridge exists, so the live read is the truth at install time; UFW rules +# persist across boots on their own, so the default only papers over the +# no-bridge-yet window at boot on a default-subnet host. +# ('|| true': under pipefail an absent bridge would kill the script here.) +GW="$(ip -4 -o addr show dev "$NET" 2>/dev/null | awk '{ split($4, a, "/"); print a[1]; exit }' || true)" +[ -n "$GW" ] || GW=10.88.0.1 if command -v ufw >/dev/null && ufw status 2>/dev/null | grep -q "Status: active"; then if ! ufw status | grep "on $NET" | grep -q "DENY"; then diff --git a/host/migrate-host.sh b/host/migrate-host.sh index e7bf238..19a2dde 100644 --- a/host/migrate-host.sh +++ b/host/migrate-host.sh @@ -3,7 +3,7 @@ # # The zero-ceremony transition is just install.sh + setup-host.sh: that leaves # a DUAL-STACK host where legacy boxes (tag user.claudebox=1, claudenet/10.87, -# claude-dev) keep working while new mints land on boxnet/10.88. This script is +# claude-dev) keep working while new mints land on boxnet. This script is # the two things that path does not do: # # migrate-host.sh --box re-home ONE legacy box onto the new stack @@ -88,17 +88,19 @@ rehome_one() { incus start "$b" >/dev/null 2>&1 || { warn "$b: did not restart — start it by hand"; return 1; } # 3. VERIFY THE EFFECT, not the exit codes (the whole repo's lesson). The box - # must be on 10.88 and actually resolve+reach the internet on its new leg - # before we call it migrated. - local _i ip + # must be on boxnet's subnet (read off the network — BOX_SUBNET moves it, + # #80) and actually resolve+reach the internet on its new leg before we + # call it migrated. + local _i ip pfx + pfx="$(incus network get boxnet ipv4.address 2>/dev/null | cut -d/ -f1)"; pfx="${pfx%.*}." ip="" for _i in $(seq 1 30); do ip="$(incus exec "$b" -- ip -4 -o addr show scope global /dev/null \ - | awk '{for(i=1;i/dev/null 2>&1; then # LAST, and only once the move is VERIFIED: drop the legacy tag. Until this # point the box wears both tags, so a failure anywhere above leaves it a diff --git a/host/setup-host.sh b/host/setup-host.sh index dff7171..ea0d636 100755 --- a/host/setup-host.sh +++ b/host/setup-host.sh @@ -45,6 +45,88 @@ else exit 1 fi +# --- The subnet, and the refusal to build on one something already owns ----- +# (#80.) The stack's subnet was hardcoded, and running setup-host INSIDE a box +# gave the guest a nested boxnet claiming the exact subnet and gateway of its +# own uplink: the guest then held its gateway's address as a LOCAL address, +# carried two connected routes for the subnet, and suffered intermittent, +# self-recovering egress blackouts nobody could attribute — the host looked +# clean the whole time. The flagship use case funnels agents toward doing +# exactly this (working on box, in a box), so the guard must refuse BEFORE +# any mutation, and name the way out (BOX_SUBNET). + +# BOX_SUBNET must be a /24 with a zero host octet — a.b.c.0/24. Everything +# the stack derives (the bridge address, the gateway carve-out, the firewall) +# assumes that shape, and a garbage value must die HERE, never inside an +# incus create or an nft rule. +valid_subnet() { + local o a="" b="" c="" rest="" + case "$1" in *.0/24) ;; *) return 1 ;; esac + IFS=. read -r a b c rest <<<"${1%/24}" + [ "$rest" = 0 ] || return 1 + for o in "$a" "$b" "$c"; do + case "$o" in ''|*[!0-9]*) return 1 ;; esac + [ "${#o}" -le 3 ] && [ "$o" -le 255 ] || return 1 + done +} + +# Who, other than box's own bridge, already owns an address inside $1? +# Prints the claimant and succeeds when the subnet is claimed by a FOREIGNER; +# stays silent and fails when it is free — or held only by boxnet, which is +# the legitimate re-run, converging a stack this script built before. The +# most telling claimant is the default route's gateway: if it sits inside the +# target subnet, this machine's own uplink lives there — i.e. this is almost +# certainly the inside of a box. Pure over `ip` output, so test/cli.sh can +# drive it against canned tables with a shim ip. +subnet_claimant() { + local pfx hit + pfx="${1%0/24}" + hit="$(ip -4 route show default 2>/dev/null | awk -v p="$pfx" ' + { gw = ""; dev = "" + for (i = 1; i < NF; i++) { if ($i == "via") gw = $(i+1); if ($i == "dev") dev = $(i+1) } + if (index(gw, p) == 1 && dev != "boxnet") { + print "this machine\047s own DEFAULT GATEWAY (" gw " via " dev ")"; exit } }')" + if [ -z "$hit" ]; then + hit="$(ip -4 -o addr show 2>/dev/null | awk -v p="$pfx" ' + $2 != "boxnet" && index($4, p) == 1 { print "interface " $2 " (" $4 ")"; exit }')" + fi + [ -n "$hit" ] && printf '%s\n' "$hit" +} + +BOX_SUBNET="${BOX_SUBNET:-10.88.0.0/24}" +if ! valid_subnet "$BOX_SUBNET"; then + echo "ERROR: BOX_SUBNET='$BOX_SUBNET' is not a sane subnet — the stack takes a" >&2 + echo " /24 with a zero host octet, e.g. BOX_SUBNET=10.89.0.0/24" >&2 + exit 1 +fi +BOX_GW="${BOX_SUBNET%.0/24}.1" + +if hit="$(subnet_claimant "$BOX_SUBNET")"; then + echo "ERROR: refusing to build boxnet on $BOX_SUBNET — that subnet is already" >&2 + echo " claimed here by $hit." >&2 + echo " If that is this machine's uplink, you are INSIDE a box: a nested" >&2 + echo " stack on the guest's own subnet captures its gateway address and" >&2 + echo " blackholes its egress, intermittently (issue #80)." >&2 + echo " Nothing was changed. To build a nested stack anyway, pick a free" >&2 + echo " subnet: BOX_SUBNET=10.89.0.0/24 box setup-host" >&2 + exit 1 +fi + +# A bridge this script built before is the one claimant that is NOT a +# collision — but it must AGREE with the target: setup-host converges an +# existing bridge, it never re-addresses one (boxes hold leases on it). +# ('|| true': under pipefail, `ip … dev boxnet` on a fresh host — no such +# device — would kill the script right here instead of answering "no bridge".) +have_gw="$(ip -4 -o addr show dev boxnet 2>/dev/null | awk '{ split($4, a, "/"); print a[1]; exit }' || true)" +if [ -n "$have_gw" ] && [ "$have_gw" != "$BOX_GW" ]; then + echo "ERROR: boxnet already exists on ${have_gw%.*}.0/24 and the target is $BOX_SUBNET —" >&2 + echo " setup-host converges an existing bridge, it never re-addresses one." >&2 + echo " Re-run with the bridge's own subnet:" >&2 + echo " BOX_SUBNET=${have_gw%.*}.0/24 box setup-host" >&2 + echo " (or move the bridge first: incus network set boxnet ipv4.address $BOX_GW/24)" >&2 + exit 1 +fi + # apt, unattended-safe. install.sh now runs us without a human watching, and # a fresh cloud image has apt-daily/unattended-upgrades holding the dpkg lock # for the first minutes of its life — plain 'apt-get install' then waits on it @@ -145,22 +227,45 @@ PRESEED fi # Isolated NAT network. IPv6 off: one less egress path to reason about. -# 10.88, not 10.87: a pre-rename host may still carry claudenet on 10.87 with -# legacy boxes attached — two bridges must not claim one subnet. +# The default is 10.88 — not 10.87: a pre-rename host may still carry +# claudenet on 10.87 with legacy boxes attached — two bridges must not claim +# one subnet. BOX_SUBNET (validated and cleared by the #80 guard above) picks +# another /24; the gateway and every rule below derive from it. incus network show boxnet >/dev/null 2>&1 || incus network create boxnet \ - ipv4.address=10.88.0.1/24 ipv4.nat=true ipv6.address=none + ipv4.address="$BOX_GW/24" ipv4.nat=true ipv6.address=none # ACL: default egress allow (internet), explicit drops for private space. -# Gateway carve-out first so instance DNS (dnsmasq on 10.88.0.1) survives. -if ! incus network acl show box-isolate >/dev/null 2>&1; then - incus network acl create box-isolate - incus network acl rule add box-isolate egress action=allow destination=10.88.0.1/32 - incus network acl rule add box-isolate egress action=drop destination=10.0.0.0/8 - incus network acl rule add box-isolate egress action=drop destination=172.16.0.0/12 - incus network acl rule add box-isolate egress action=drop destination=192.168.0.0/16 - incus network acl rule add box-isolate egress action=drop destination=169.254.0.0/16 - incus network acl rule add box-isolate egress action=drop destination=100.64.0.0/10 -fi +# Gateway carve-out first so instance DNS (dnsmasq on the gateway) survives. +# 'edit' the full shipped ruleset, not create-once: the carve-out derives +# from BOX_SUBNET now, and a bridge moved off a colliding subnet (#80's +# escape hatch) left the OLD /32 behind — box DNS to the new gateway then +# died inside the 10.0.0.0/8 drop, looking like a dead resolver, not a stale +# ACL. A conditional 'rule add' cannot converge that (the stale carve-out +# would survive beside the new one); replacing the ruleset does, idempotently. +incus network acl show box-isolate >/dev/null 2>&1 || incus network acl create box-isolate +incus network acl edit box-isolate < Date: Sat, 18 Jul 2026 19:47:20 +0000 Subject: [PATCH 2/4] feat(doctor): recognize the #80 gateway-squat signature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The doctor had already CAUGHT #80 without knowing it: 'DIRTY cannot reach 1.1.1.1 by address' beside 'OK resolves deb.debian.org' — and attributed it to the host admin, whose every check was green. That split IS the fingerprint: the impostor dnsmasq on the captured gateway address keeps answering names (it forwards upstream via the default route) while direct IP egress dies. Three additions: · gw_squat_signature — pure text (ip -4 route show / ip -4 -o addr show) in, findings out: a default gateway held as a LOCAL address (the kernel's local table then eats packets meant for the real gateway — DNS, unicast DHCP renewals), and duplicate connected routes for the uplink subnet (whichever link last gains carrier wins, and a nested bridge with carrier blackholes egress). Pure so test/cli.sh drives it against synthetic tables, including the issue's exact captures. · judged on THIS machine first, both tiers, before any daemon call — inside a poisoned guest the daemon answering is the nested impostor, happily judging its own stack clean — and probed INSIDE every box the doctor examines (probe_sig), where the signature actually lives. · the admin ACL section now compares the gateway carve-out to boxnet's actual gateway: the #80 escape hatch moves the bridge, and the stale /32 strands box DNS inside the 10.0.0.0/8 drop. Measured live on a workaround host: doctor names the stale carve-out and the BOX_SUBNET re-run that converges it. The egress-broken-but-DNS-fine hint now names itself on both tiers and points at the signature probes for the verdict. Co-Authored-By: Claude Fable 5 --- drill/doctor.sh | 112 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 106 insertions(+), 6 deletions(-) diff --git a/drill/doctor.sh b/drill/doctor.sh index f8de671..4fac72b 100755 --- a/drill/doctor.sh +++ b/drill/doctor.sh @@ -35,7 +35,75 @@ no() { printf ' \033[31mDIRTY\033[0m %s\n' "$*"; bad=$((bad + 1)); } inf() { printf ' %s\n' "$*"; } head_() { printf '\n\033[1m%s\033[0m\n' "$*"; } +# --- The #80 signature: a nested box stack squatting on the gateway --------- +# setup-host run INSIDE a box builds a nested boxnet on the guest's own uplink +# subnet. The measured mechanism, and the two lines this function reads for: +# hold your own gateway's address and the kernel's local table eats packets +# meant for the real gateway (DNS, unicast DHCP renewals); carry two connected +# routes for the uplink subnet and whichever link last has carrier wins — a +# nested bridge gaining carrier blackholes egress instantly. Pure text in, +# findings out (one per line, silence is clean), so test/cli.sh drives it +# against synthetic route tables and the guest probe can feed it routes read +# INSIDE a box. Inputs: `ip -4 route show` and `ip -4 -o addr show` output. +gw_squat_signature() { + local routes="$1" addrs="$2" gw updev + gw="$(printf '%s\n' "$routes" | awk '$1 == "default" { for (i = 1; i < NF; i++) if ($i == "via") { print $(i+1); exit } }')" + updev="$(printf '%s\n' "$routes" | awk '$1 == "default" { for (i = 1; i < NF; i++) if ($i == "dev") { print $(i+1); exit } }')" + [ -n "$gw" ] || return 0 # no default route: nothing to squat on + printf '%s\n' "$addrs" | awk -v gw="$gw" ' + { split($4, a, "/") + if (a[1] == gw) { print "the default gateway " gw " is held as a LOCAL address (on " $2 ") — the kernel delivers packets meant for the gateway to this machine itself"; exit } }' + printf '%s\n' "$routes" | awk -v updev="$updev" ' + / proto kernel / && $2 == "dev" { + cnt[$1]++; devs[$1] = devs[$1] (devs[$1] ? ", " : "") $3 + if ($3 == updev) up = $1 + } + END { if (up != "" && cnt[up] > 1) + print "duplicate connected routes for the uplink subnet " up " (" devs[up] ") — whichever link last gains carrier wins, and a nested bridge with carrier blackholes egress" }' +} + +# The signature, probed INSIDE a box: its routes, read where they live. A +# poisoned guest looks healthy from every host-side config check — the nested +# bridge and the captured gateway exist only in the guest's kernel. +probe_sig() { + local b="$1" routes addrs sig line + routes="$(timeout -k 5 20 incus exec "$b" -- ip -4 route show /dev/null)" + addrs="$(timeout -k 5 20 incus exec "$b" -- ip -4 -o addr show /dev/null)" + if [ -z "$routes" ]; then + inf "could not read routes inside '$b' — the #80 signature was not probed" + return 0 + fi + sig="$(gw_squat_signature "$routes" "$addrs")" + if [ -n "$sig" ]; then + while IFS= read -r line; do no "inside '$b': $line"; done <<<"$sig" + inf "a box stack was installed INSIDE this box — its nested bridge claims the" + inf "box's own uplink subnet, and egress blacks out intermittently (issue #80)." + inf "fix, inside the box: sudo incus network set boxnet ipv4.address 10.89.0.1/24" + inf " (or remove the nested stack there: box teardown-host)" + else + ok "no #80 signature inside '$b' — nothing is squatting on its gateway" + fi +} + command -v incus >/dev/null || { echo "doctor: incus is not installed on this host."; exit 1; } + +# THIS MACHINE first, both tiers, before anything that needs the daemon: the +# #80 signature is a fact about the kernel's routing tables, not about incus — +# and a poisoned guest is exactly where the daemon answering below may be the +# WRONG (nested) one, judging its own impostor stack clean. +head_ "This machine — is a nested box stack squatting on the gateway? (#80)" +sig="$(gw_squat_signature "$(ip -4 route show 2>/dev/null)" "$(ip -4 -o addr show 2>/dev/null)")" +if [ -n "$sig" ]; then + while IFS= read -r line; do no "$line"; done <<<"$sig" + inf "a box stack was built on a machine whose uplink already owns its subnet —" + inf "run inside a box, that is issue #80: egress blacks out intermittently while" + inf "everything looks healthy. setup-host now refuses this; this machine already has it." + inf "fix: move the nested bridge off the uplink's subnet:" + inf " sudo incus network set boxnet ipv4.address 10.89.0.1/24" + inf " (or remove the nested stack: box teardown-host)" +else + ok "the default gateway is not held locally, and the uplink subnet has one connected route" +fi timeout 10 incus list >/dev/null 2>&1 || { echo "doctor: the incus daemon is not answering (see issue #26 for recovery):" echo " sudo pkill -9 -f 'incusd shutdown'" @@ -85,12 +153,21 @@ if [ "$TIER" = restricted ]; then | awk -F, '$2 == "RUNNING" { print $1; exit }')" if [ -n "$probe" ]; then inf "probing inside '$probe':" - timeout -k 5 25 incus exec "$probe" -- curl -sS -m 10 -o /dev/null https://1.1.1.1 /dev/null \ - && ok "reaches 1.1.1.1 by address — egress routing is fine" \ - || no "cannot reach 1.1.1.1 by address — egress routing is broken (an admin problem: box doctor as admin)" - timeout -k 5 25 incus exec "$probe" -- getent hosts deb.debian.org /dev/null 2>&1 \ - && ok "resolves deb.debian.org — DNS works" \ - || no "CANNOT resolve deb.debian.org — an admin problem (the resolver pin lives on the host): box doctor as admin" + if timeout -k 5 25 incus exec "$probe" -- curl -sS -m 10 -o /dev/null https://1.1.1.1 /dev/null; then + routing=1; ok "reaches 1.1.1.1 by address — egress routing is fine" + else + routing=0; no "cannot reach 1.1.1.1 by address — egress routing is broken (an admin problem: box doctor as admin)" + fi + if timeout -k 5 25 incus exec "$probe" -- getent hosts deb.debian.org /dev/null 2>&1; then + ok "resolves deb.debian.org — DNS works" + # Egress broken while DNS resolves is #80's fingerprint: an impostor + # dnsmasq on a captured gateway address answers names happily (it + # forwards upstream via the default route) while direct IP egress dies. + [ "$routing" = 0 ] && inf "…egress broken while DNS resolves is #80's fingerprint — the signature probe below answers whether something inside this box squats on its gateway" + else + no "CANNOT resolve deb.debian.org — an admin problem (the resolver pin lives on the host): box doctor as admin" + fi + probe_sig "$probe" else inf "no running box to probe with (mint one: box new --name work)" fi @@ -209,6 +286,22 @@ if incus network acl show box-isolate >/dev/null 2>&1; then no "an @internal rule survived phase D" [ "$FIX" = 1 ] && { incus network acl rule remove box-isolate egress action=drop destination=@internal && inf "reverted: @internal rule removed"; } fi + # The gateway carve-out must track the BRIDGE. #80's escape hatch moves + # boxnet off a colliding subnet — and the stale /32 then strands box DNS + # inside the 10.0.0.0/8 drop, which presents as a dead resolver, never as + # a stale ACL. Compare the allow rule to boxnet's actual gateway. + gwaddr="$(incus network get boxnet ipv4.address 2>/dev/null | cut -d/ -f1)" + carve="$(incus network acl show box-isolate 2>/dev/null \ + | awk '/- action: allow/ { hit = 1; next } hit && /destination:/ { sub("/32", "", $2); print $2; exit } { hit = 0 }')" + if [ -n "$gwaddr" ] && [ -n "$carve" ]; then + if [ "$carve" = "$gwaddr" ]; then + ok "the gateway carve-out matches boxnet's gateway ($gwaddr) — box DNS survives the 10/8 drop" + else + no "the gateway carve-out ($carve/32) does NOT match boxnet's gateway ($gwaddr) — box DNS to the gateway dies inside the 10.0.0.0/8 drop" + inf "the bridge moved (#80's escape hatch) and the ACL did not follow" + inf "fix: BOX_SUBNET=${gwaddr%.*}.0/24 box setup-host (it converges the ACL now)" + fi + fi else inf "box-isolate does not exist (a fresh host)" fi @@ -329,6 +422,11 @@ if [ -n "$probe" ] && [ "$FIX" != 1 ]; then if timeout -k 5 25 incus exec "$probe" -- getent hosts deb.debian.org /dev/null 2>&1; then ok "resolves deb.debian.org — DNS works" + # The OTHER split from the one below: egress broken while DNS resolves is + # #80's fingerprint — an impostor dnsmasq on a captured gateway address + # keeps answering names (it forwards upstream via the default route) + # while direct IP egress dies. The signature probe underneath answers it. + [ "$routing" = 0 ] && inf "…egress broken while DNS resolves is #80's fingerprint — see the signature probe below" else no "CANNOT resolve deb.debian.org — this is exactly what kills cloud-init on every cold mint" # Egress by address was probed above. If it worked, the fault is purely @@ -341,6 +439,8 @@ if [ -n "$probe" ] && [ "$FIX" != 1 ]; then inf "…and it cannot reach 1.1.1.1 by address either — so egress itself is broken, not just DNS." fi fi + + probe_sig "$probe" else inf "no box to probe with (mint one, or run without --fix after a run)" fi -- 2.45.2 From 7d48822cc0a933d68516f21efde79a5b55b73a5e Mon Sep 17 00:00:00 2001 From: dan-claude-bot Date: Sat, 18 Jul 2026 19:47:20 +0000 Subject: [PATCH 3/4] test(cli): drive the #80 guard, the BOX_SUBNET plumb-through, and the signature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two pure functions are extracted and driven, the same seam as box_tier and load_template: a shim ip serves canned route tables (the poisoned guest verbatim from #80's capture, the remapped-bridge workaround, a healthy stack host), and each signature line is proven to fire alone and to stay silent on the clean states. Then the WHOLE setup-host is driven end to end under shims (fake incus and sudo log every call; fake id keeps it unprivileged): the three refusals (gateway-in-subnet, foreign interface, garbage/wrong-shape BOX_SUBNET) exit 1 having made NO incus and NO sudo call — refuse-before-mutation is asserted on the logs' absence, not assumed — the existing-bridge mismatch refuses, the legitimate converge (boxnet's own prior claim) still reaches 'Host ready', and a fresh BOX_SUBNET=10.89.0.0/24 build lands ipv4.address=10.89.0.1/24 on the bridge and destination: 10.89.0.1/32 in the ACL, proving every derived value follows the one input. Line-order guards pin the wiring: the subnet guard precedes the first mutation in setup-host, the machine-local signature precedes the doctor's daemon checks, probe_sig runs on both tiers, box-firewall reads the gateway off the live bridge, and the drill/multiuser/migrate probes derive their prefix from the network. Co-Authored-By: Claude Fable 5 --- test/cli.sh | 246 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 246 insertions(+) diff --git a/test/cli.sh b/test/cli.sh index 78f793c..f16b5d6 100644 --- a/test/cli.sh +++ b/test/cli.sh @@ -458,6 +458,252 @@ check "revoke: the absence assert covers the incus-user state too" 0 "" \ check "revoke: the state checks go through \$SUDO test (an unprivileged stat lies)" 0 "" \ grep -qF '$SUDO test -d "/var/lib/incus/users/$uid"' "$ROOT/host/revoke-user.sh" +# --------------------------------------------------------------------------- +# The #80 guard and BOX_SUBNET. setup-host run inside a box used to build a +# nested boxnet on the guest's own uplink subnet — captured gateway, duplicate +# routes, intermittent egress blackouts. The guard's two pure functions are +# extracted and DRIVEN (a shim ip serves canned route tables, the same seam as +# the shim id), and then the WHOLE script is driven end to end under shims: +# the refusal paths must exit 1 having touched nothing (the incus/sudo shims +# log every call, and the log must not exist), the converge path must still +# run, and BOX_SUBNET must plumb through to every derived value. +# --------------------------------------------------------------------------- +cat > "$SHIMDIR/ip" <<'SHIM' +#!/usr/bin/env bash +# Fake `ip`: canned tables for the #80 guard and signature — just the reads +# setup-host and doctor make. Specific patterns first: case takes the first hit. +case "$*" in + "-4 -o addr show dev boxnet") printf '%s\n' "${FAKE_IP4_BOXNET:-}" ;; + "-4 route show default") printf '%s\n' "${FAKE_IP4_DEFAULT:-}" ;; + "-4 route show") printf '%s\n' "${FAKE_IP4_ROUTES:-}" ;; + "-4 -o addr show") printf '%s\n' "${FAKE_IP4_ADDRS:-}" ;; +esac +exit 0 +SHIM +chmod +x "$SHIMDIR/ip" + +# The route tables, verbatim from issue #80's capture (the poisoned guest) and +# from the states around it. +D_INBOX='default via 10.88.0.1 dev enp5s0 proto dhcp src 10.88.0.202 metric 1024' +D_LAN='default via 192.168.1.1 dev eno1 proto dhcp metric 100' +A_GUEST='2: enp5s0 inet 10.88.0.202/24 metric 1024 brd 10.88.0.255 scope global dynamic enp5s0' +A_HOSTSTACK='2: eno1 inet 192.168.1.50/24 brd 192.168.1.255 scope global dynamic eno1 +5: boxnet inet 10.88.0.1/24 scope global boxnet' +A_FOREIGN='2: eno1 inet 192.168.1.50/24 brd 192.168.1.255 scope global dynamic eno1 +3: virbr7 inet 10.88.0.7/24 brd 10.88.0.255 scope global virbr7' + +SUBFN="$(mktemp)" +awk '/^valid_subnet\(\) \{/,/^\}/' "$ROOT/host/setup-host.sh" > "$SUBFN" +check "valid_subnet: extracted from setup-host.sh (guards the awk)" 0 "return 1" cat "$SUBFN" +check "valid_subnet: the extracted function is valid bash" 0 "" bash -n "$SUBFN" +vsub() { bash -c ". '$SUBFN'; valid_subnet \"\$1\"" _ "$1"; } +check "valid_subnet: the default is valid" 0 "" vsub 10.88.0.0/24 +check "valid_subnet: the documented escape hatch is valid" 0 "" vsub 10.89.0.0/24 +check "valid_subnet: any a.b.c.0/24 is valid" 0 "" vsub 192.168.7.0/24 +check "valid_subnet: not-a-/24 is refused" 1 "" vsub 10.88.0.0/16 +check "valid_subnet: a nonzero host octet is refused" 1 "" vsub 10.88.0.5/24 +check "valid_subnet: an octet past 255 is refused" 1 "" vsub 300.88.0.0/24 +check "valid_subnet: a bare address is refused" 1 "" vsub 10.88.0.0 +check "valid_subnet: garbage is refused" 1 "" vsub banana +check "valid_subnet: an empty value is refused" 1 "" vsub "" +rm -f "$SUBFN" + +CLMFN="$(mktemp)" +awk '/^subnet_claimant\(\) \{/,/^\}/' "$ROOT/host/setup-host.sh" > "$CLMFN" +check "subnet_claimant: extracted from setup-host.sh (guards the awk)" 0 "DEFAULT GATEWAY" cat "$CLMFN" +check "subnet_claimant: the extracted function is valid bash" 0 "" bash -n "$CLMFN" +claim() { # claim + FAKE_IP4_DEFAULT="$2" FAKE_IP4_ADDRS="$3" PATH="$SHIMDIR:$PATH" \ + bash -c ". '$CLMFN'; subnet_claimant \"\$1\"" _ "$1" +} +check "claimant: the default gateway inside the target is the smoking gun" \ + 0 "DEFAULT GATEWAY" claim 10.88.0.0/24 "$D_INBOX" "$A_GUEST" +check "claimant: a foreign interface inside the target is named" \ + 0 "virbr7" claim 10.88.0.0/24 "$D_LAN" "$A_FOREIGN" +check "claimant: boxnet's own prior claim is the converge path — CLEAN" \ + 1 "" claim 10.88.0.0/24 "$D_LAN" "$A_HOSTSTACK" +check "claimant: a free subnet is clean" \ + 1 "" claim 10.89.0.0/24 "$D_LAN" "$A_HOSTSTACK" +check "claimant: 10.8.0.0/24 does not prefix-match 10.88.x (the dot terminates)" \ + 1 "" claim 10.8.0.0/24 "$D_INBOX" "$A_GUEST" +rm -f "$CLMFN" + +# --- the whole script, driven: refuse-before-mutation, converge, plumb-through +SETUPSHIM="$(mktemp -d)" +cat > "$SETUPSHIM/incus" <<'SHIM' +#!/usr/bin/env bash +# Fake incus for the driven setup-host: records every call (and, for the +# stdin verbs, the stdin) to $FAKE_INCUS_LOG, answers the existence probes +# from FAKE_HAVE_*, and never goes near a daemon. +[ -n "${FAKE_INCUS_LOG:-}" ] && printf 'incus %s\n' "$*" >> "$FAKE_INCUS_LOG" +case "$*" in + *"admin init --preseed"*|*"acl edit"*|*"profile edit"*) + if [ -n "${FAKE_INCUS_LOG:-}" ]; then sed 's/^/ | /' >> "$FAKE_INCUS_LOG"; else cat >/dev/null; fi ;; +esac +case "$*" in + "storage show default") [ -n "${FAKE_HAVE_STORAGE:-}" ] || exit 1 ;; + "network show boxnet") [ -n "${FAKE_HAVE_BOXNET:-}" ] || exit 1 ;; + "network acl show box-isolate") [ -n "${FAKE_HAVE_ACL:-}" ] || exit 1 ;; + "profile show box-net") [ -n "${FAKE_HAVE_PROFILE:-}" ] || exit 1 ;; +esac +exit 0 +SHIM +cat > "$SETUPSHIM/sudo" <<'SHIM' +#!/usr/bin/env bash +# Fake sudo: logs to $FAKE_SUDO_LOG and swallows everything — the driven +# setup-host must never mutate the machine running this suite. +[ -n "${FAKE_SUDO_LOG:-}" ] && printf 'sudo %s\n' "$*" >> "$FAKE_SUDO_LOG" +exit 0 +SHIM +chmod +x "$SETUPSHIM/incus" "$SETUPSHIM/sudo" + +runsetup() { # runsetup [VAR=val ...] — the real setup-host, under shims + env FAKE_UID=1000 FAKE_GROUPS="users incus-admin" \ + PATH="$SETUPSHIM:$SHIMDIR:$PATH" "$@" bash "$ROOT/host/setup-host.sh" +} + +W80="$(mktemp -d)" +# Refusal 1: the default gateway sits inside the target — the inside of a box. +check "setup-host: gw-in-subnet REFUSES and names issue #80" 1 "issue #80" \ + runsetup FAKE_IP4_DEFAULT="$D_INBOX" FAKE_IP4_ADDRS="$A_GUEST" \ + FAKE_INCUS_LOG="$W80/g1.log" FAKE_SUDO_LOG="$W80/s1.log" +check "setup-host: ...naming BOX_SUBNET as the way out" 1 "BOX_SUBNET" \ + runsetup FAKE_IP4_DEFAULT="$D_INBOX" FAKE_IP4_ADDRS="$A_GUEST" +check "setup-host: the refusal made NO incus call (refuse precedes mutation)" 1 "" \ + test -e "$W80/g1.log" +check "setup-host: the refusal made NO sudo call either" 1 "" \ + test -e "$W80/s1.log" +# Refusal 2: a foreign interface owns an address inside the target. +check "setup-host: a foreign interface in the subnet REFUSES" 1 "virbr7" \ + runsetup FAKE_IP4_DEFAULT="$D_LAN" FAKE_IP4_ADDRS="$A_FOREIGN" +# Refusal 3: garbage BOX_SUBNET dies at the gate. +check "setup-host: a garbage BOX_SUBNET is refused by name" 1 "not a sane subnet" \ + runsetup BOX_SUBNET=banana +check "setup-host: a /16 BOX_SUBNET is refused" 1 "not a sane subnet" \ + runsetup BOX_SUBNET=10.88.0.0/16 +# Refusal 4: an existing bridge on ANOTHER subnet is never re-addressed. +check "setup-host: a bridge on another subnet refuses (converge, don't re-address)" \ + 1 "never re-addresses" \ + runsetup FAKE_IP4_DEFAULT="$D_LAN" FAKE_IP4_ADDRS="$A_HOSTSTACK" \ + FAKE_IP4_BOXNET='5: boxnet inet 10.89.0.1/24 scope global boxnet' \ + BOX_SUBNET=10.88.0.0/24 +# The legitimate re-run: boxnet itself owns the subnet — setup-host converges. +check "setup-host: a prior boxnet claiming the subnet CONVERGES (no false positive)" \ + 0 "Host ready" \ + runsetup FAKE_IP4_DEFAULT="$D_LAN" FAKE_IP4_ADDRS="$A_HOSTSTACK" \ + FAKE_IP4_BOXNET='5: boxnet inet 10.88.0.1/24 scope global boxnet' \ + FAKE_HAVE_STORAGE=1 FAKE_HAVE_BOXNET=1 FAKE_HAVE_ACL=1 FAKE_HAVE_PROFILE=1 +# BOX_SUBNET plumbs through: a fresh build on 10.89.0.0/24 must derive EVERY +# value from it — the bridge address and the ACL's gateway carve-out. +check "setup-host: BOX_SUBNET drives a fresh build to completion" 0 "Host ready" \ + runsetup BOX_SUBNET=10.89.0.0/24 FAKE_IP4_DEFAULT="$D_INBOX" FAKE_IP4_ADDRS="$A_GUEST" \ + FAKE_INCUS_LOG="$W80/g2.log" FAKE_SUDO_LOG="$W80/s2.log" +check "setup-host: ...the bridge derives from BOX_SUBNET" 0 "" \ + grep -qF 'network create boxnet ipv4.address=10.89.0.1/24' "$W80/g2.log" +check "setup-host: ...and so does the ACL's gateway carve-out" 0 "" \ + grep -qF 'destination: 10.89.0.1/32' "$W80/g2.log" +# ...which also proves the guard scans the TARGET subnet: the same tables that +# refused the default (gw 10.88.0.1) pass once BOX_SUBNET moves off it — the +# issue's workaround host, sanctioned. +rm -rf "$W80" "$SETUPSHIM" + +# The guard must be the FIRST effective act — before the incus install, the +# usermod, every apt call. Line order, fail-closed on either grep missing. +# shellcheck disable=SC2016 # the $-strings are literals in the target file +check "setup-host: the subnet guard precedes the first mutation" 0 "" bash -c ' + guard="$(grep -n "subnet_claimant \"\$BOX_SUBNET\"" "'"$ROOT"'/host/setup-host.sh" | head -1 | cut -d: -f1)" + mut="$(grep -n "^if ! command -v incus" "'"$ROOT"'/host/setup-host.sh" | head -1 | cut -d: -f1)" + [ -n "$guard" ] && [ -n "$mut" ] && [ "$guard" -lt "$mut" ]' +# box-firewall follows the bridge, wherever BOX_SUBNET put it. +# shellcheck disable=SC2016 # the $-string is a literal in the target file +check "box-firewall: the gateway is read off the live bridge, not hardcoded" 0 "" \ + grep -qF 'addr show dev "$NET"' "$ROOT/host/box-firewall.sh" +# The drill and migrate probes derive the prefix from the network — a +# BOX_SUBNET host must not fail its own rehearsals. +check "drill: derives the boxnet prefix from the network" 0 "" \ + grep -qF 'network get boxnet ipv4.address' "$ROOT/drill/drill.sh" +check "multiuser: derives the boxnet prefix from the network" 0 "" \ + grep -qF 'network get boxnet ipv4.address' "$ROOT/drill/multiuser.sh" +check "migrate-host: derives the boxnet prefix from the network" 0 "" \ + grep -qF 'network get boxnet ipv4.address' "$ROOT/host/migrate-host.sh" + +# --------------------------------------------------------------------------- +# The doctor's #80 signature. gw_squat_signature is pure text → findings, so +# it is extracted and driven against synthetic route tables — including the +# EXACT poisoned state from the issue, the workaround state (bridge remapped: +# clean), and a healthy host running the stack (clean). +# --------------------------------------------------------------------------- +SIGFN="$(mktemp)" +awk '/^gw_squat_signature\(\) \{/,/^\}/' "$ROOT/drill/doctor.sh" > "$SIGFN" +check "gw_squat_signature: extracted from doctor.sh (guards the awk)" 0 "default" cat "$SIGFN" +check "gw_squat_signature: the extracted function is valid bash" 0 "" bash -n "$SIGFN" +sig() { bash -c ". '$SIGFN'; gw_squat_signature \"\$1\" \"\$2\"" _ "$1" "$2"; } +nosig() { [ -z "$(sig "$1" "$2")" ]; } + +# The poisoned guest, verbatim from #80: gateway held locally AND duplicated +# connected routes for the uplink subnet. +R_POISON="$D_INBOX +10.88.0.0/24 dev boxnet proto kernel scope link src 10.88.0.1 linkdown +10.88.0.0/24 dev enp5s0 proto kernel scope link src 10.88.0.202 metric 1024 +10.88.0.1 dev enp5s0 proto dhcp scope link src 10.88.0.202 metric 1024" +A_POISON="$A_GUEST +17: boxnet inet 10.88.0.1/24 scope global boxnet" +check "signature: poisoned guest — the gateway is held as a LOCAL address" \ + 0 "held as a LOCAL address" sig "$R_POISON" "$A_POISON" +check "signature: poisoned guest — duplicate connected routes for the uplink" \ + 0 "duplicate connected routes" sig "$R_POISON" "$A_POISON" +# The workaround state (#80's fix: bridge remapped off the uplink subnet) — +# both signature lines must be ABSENT. +R_REMAP="$D_INBOX +10.88.0.0/24 dev enp5s0 proto kernel scope link src 10.88.0.202 metric 1024 +10.88.0.1 dev enp5s0 proto dhcp scope link src 10.88.0.202 metric 1024 +10.89.0.0/24 dev boxnet proto kernel scope link src 10.89.0.1 linkdown" +A_REMAP="$A_GUEST +17: boxnet inet 10.89.0.1/24 scope global boxnet" +check "signature: the remapped-bridge workaround is CLEAN" 0 "" nosig "$R_REMAP" "$A_REMAP" +# A healthy HOST running the stack: boxnet legitimately owns its subnet, and +# the uplink is elsewhere — clean, or every host would cry wolf. +R_HOST="$D_LAN +192.168.1.0/24 dev eno1 proto kernel scope link src 192.168.1.50 +10.88.0.0/24 dev boxnet proto kernel scope link src 10.88.0.1" +check "signature: a healthy host running the stack is CLEAN" 0 "" nosig "$R_HOST" "$A_HOSTSTACK" +check "signature: no default route → nothing to judge (clean)" 0 "" \ + nosig "10.88.0.0/24 dev boxnet proto kernel scope link src 10.88.0.1" "$A_HOSTSTACK" +# Each line fires on its own: a captured gateway without duplicate routes... +R_GWONLY="$D_INBOX +10.88.0.0/24 dev enp5s0 proto kernel scope link src 10.88.0.202 metric 1024" +check "signature: a captured gateway alone still fires" \ + 0 "held as a LOCAL address" sig "$R_GWONLY" "$A_POISON" +# ...and duplicate routes without the gateway captured (nested bridge on .5). +A_DUPONLY="$A_GUEST +17: boxnet inet 10.88.0.5/24 scope global boxnet" +check "signature: duplicate routes alone still fire" \ + 0 "duplicate connected routes" sig "$R_POISON" "$A_DUPONLY" +rm -f "$SIGFN" + +# The wiring: the signature is judged on THIS machine before any daemon call +# (the daemon answering could be the nested impostor), probed INSIDE boxes on +# both tiers, and the egress-broken-DNS-fine split names the fingerprint. +# shellcheck disable=SC2016 # the $-strings are literals in the target file +check "doctor: this machine's signature precedes the daemon checks" 0 "" bash -c ' + sig="$(grep -n "is a nested box stack squatting" "'"$ROOT"'/drill/doctor.sh" | head -1 | cut -d: -f1)" + daemon="$(grep -n "timeout 10 incus list" "'"$ROOT"'/drill/doctor.sh" | head -1 | cut -d: -f1)" + [ -n "$sig" ] && [ -n "$daemon" ] && [ "$sig" -lt "$daemon" ]' +# shellcheck disable=SC2016 # the $-string is a literal in the target file +check "doctor: the signature is probed inside boxes on BOTH tiers" 0 "" bash -c ' + [ "$(grep -c "probe_sig \"\$probe\"" "'"$ROOT"'/drill/doctor.sh")" -eq 2 ]' +# shellcheck disable=SC2016 # the $-string is a literal in the target file +check "doctor: the egress-broken-DNS-fine fingerprint is named on both tiers" 0 "" bash -c ' + [ "$(grep -c "fingerprint" "'"$ROOT"'/drill/doctor.sh")" -ge 2 ]' +check "doctor: the ACL carve-out is checked against the live gateway" 0 "" \ + grep -qF "does NOT match boxnet's gateway" "$ROOT/drill/doctor.sh" + +# The docs keep the new promises. +check "help setup-host names BOX_SUBNET" 0 "BOX_SUBNET" "$BOX" help setup-host +check "help setup-host names the refusal" 0 "REFUSES" "$BOX" help setup-host +check "help doctor names the #80 signature" 0 "#80" "$BOX" help doctor +check "README documents BOX_SUBNET" 0 "" grep -qF 'BOX_SUBNET' "$ROOT/README.md" + # --------------------------------------------------------------------------- # The versioned install (#66 → 0.7.0). BOX_INSTALL_SOURCE bypasses the network, # so these are REAL runs of install.sh against throwaway BOX_HOME/BOX_BIN -- 2.45.2 From 53dc2b90470ba60098507a8a0564845dd0be662e Mon Sep 17 00:00:00 2001 From: dan-claude-bot Date: Sat, 18 Jul 2026 19:47:20 +0000 Subject: [PATCH 4/4] docs: BOX_SUBNET, the setup-host refusal, and the doctor signature (#80) README's setup-host section documents the subnet input and the refusal; 'box help setup-host' names BOX_SUBNET and the nested-box trap; 'box help doctor' names the #80 signature it now judges; the changelog carries the full story, including that suggested fix 4 (the agent-context guard note for the templates) lands in heavy-duty/rig#31's bootstrap roles per the thin-templates split (#81). Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 28 ++++++++++++++++++++++++++++ README.md | 14 ++++++++++++++ bin/box | 17 +++++++++++++++-- 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbde0ce..03fdaa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,34 @@ which records not just what changed but what each drill run proved. ### Added +- **`setup-host` refuses a claimed subnet, and `BOX_SUBNET` picks another** + (#80) — run inside a box, `setup-host` used to build a nested `boxnet` on + the exact subnet and gateway of the guest's own uplink: the guest then held + its gateway's address as a *local* address, carried duplicate connected + routes for its uplink subnet, and suffered intermittent, self-recovering + egress blackouts that looked like flaky internet (measured live: ~24–36 s + outages, roughly hourly, with the host clean throughout). `setup-host` now + scans the target subnet **before any mutation** — the default route's + gateway inside it, or any non-`boxnet` interface holding an address in it — + and refuses, naming the way out. A prior `boxnet` owning the subnet is the + legitimate converge path and does not trip it. `BOX_SUBNET=` + (validated, alongside the existing `BOX_DNS`) moves the whole stack: the + bridge address, the ACL's gateway carve-out (now converged via + `network acl edit`, so a bridge moved off a colliding subnet no longer + strands box DNS behind a stale `/32`), the firewall (`box-firewall` reads + the gateway off the live bridge), and every drill/migrate probe that used + to hardcode `10.88`. +- **`box doctor` knows the #80 signature** — a default gateway held as a + LOCAL address, and duplicate connected routes for the uplink subnet, judged + from `ip route`/`ip addr` on the machine doctor runs on (both tiers, before + any daemon check — the nested daemon answering could be the impostor) and + probed *inside* every box it examines. The existing "egress broken but DNS + fine" split now names itself as #80's fingerprint (the impostor dnsmasq on + a captured gateway keeps resolving while IP egress dies), and the admin ACL + section verifies the gateway carve-out matches `boxnet`'s actual gateway. + The agent-context guard for the templates (suggested fix 4) lands in + heavy-duty/rig#31's bootstrap roles per the thin-templates split (#81). + - **Server-posture template keys** (#81, carved from #69) — two optional `box.env` allowlist keys. `BOX_REQUIRE_VM=1` refuses both the silent container fallback (no `/dev/kvm`, exit 1) and an explicit `--container` diff --git a/README.md b/README.md index 64b3764..71e6f2e 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,20 @@ re-apply at boot via `box-firewall.service` — no post-reboot ritual. If the host lacks `dnsmasq-base` (Debian cloud images skip Recommends): `sudo apt-get install -y dnsmasq-base`. +The stack's subnet is `10.88.0.0/24` by default; `BOX_SUBNET` picks another +`/24` (`BOX_SUBNET=10.89.0.0/24 box setup-host` — the bridge address, the +ACL's gateway carve-out and the firewall all derive from it). setup-host +**refuses to build on a subnet something already claims** — most tellingly +when this machine's own default gateway sits inside it, which means it is +being run *inside a box*: a nested `boxnet` on the guest's own uplink subnet +captures its gateway address and blackholes the guest's egress in +intermittent, maddening-to-attribute blackouts +([#80](https://github.com/heavy-duty/box/issues/80)). `BOX_SUBNET` is the +sanctioned way out for a nested or otherwise-conflicted install, and +`box doctor` recognizes the poisoned state (a gateway held as a local +address, duplicate uplink routes) on the machine it runs on and inside every +box it probes. + A host still carrying the pre-0.4.0 stack: `box migrate-host --all-boxes` re-homes each legacy box onto `boxnet` (authed state preserved), and `box migrate-host --retire-legacy` removes the old bridge and profile once no diff --git a/bin/box b/bin/box index a57d6ec..b8fd45a 100755 --- a/bin/box +++ b/bin/box @@ -459,8 +459,12 @@ EOF Answer "is this host fit to mint boxes?" from ground truth, not config claims: is the Incus daemon answering, is a dnsmasq actually serving boxnet, does the kernel's bridge port say 'isolated on', is the resolver pinned or is a -host VPN's DNS leaking into boxes, can a box actually resolve names. Every -check exists because its fault has happened — most kill a cold mint with a +host VPN's DNS leaking into boxes, can a box actually resolve names — and +the #80 nested-stack signature: a default gateway held as a LOCAL address, +or duplicate connected routes for the uplink subnet, judged on this machine +AND inside every box it probes (a box stack installed inside a box squats on +the guest's gateway and blackholes its egress, intermittently). Every check +exists because its fault has happened — most kill a cold mint with a cloud-init error that names none of them. --fix also revert what a drill run may have left behind @@ -484,7 +488,16 @@ install.sh runs it for you, so this is for re-applying by hand. One run is enough. If it has to add you to the incus-admin group it re-runs itself under that group — no re-login, no second invocation. +The stack's subnet is 10.88.0.0/24 by default; BOX_SUBNET picks another /24 +(the bridge, the gateway carve-out and the firewall all derive from it). It +REFUSES, before touching anything, where the target subnet is already claimed +— most tellingly when this machine's own default gateway sits inside it, +i.e. when you are running setup-host INSIDE a box: a nested stack on the +guest's own uplink subnet captures its gateway address and blackholes its +egress, intermittently (issue #80). The sanctioned way to nest: + box setup-host + BOX_SUBNET=10.89.0.0/24 box setup-host # nested, or a conflicted host Multi-user hosts: setup-host builds the stack once, for everyone. An admin then hands individual users the restricted tier with 'box grant ' — -- 2.45.2