From b429f5b65c99e70db74b7ca12a6f0392728465a9 Mon Sep 17 00:00:00 2001 From: dan-claude-bot Date: Sat, 18 Jul 2026 19:46:58 +0000 Subject: [PATCH] feat(setup-host): refuse a claimed subnet, and BOX_SUBNET to move off one (#80) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run inside a box, setup-host built a nested boxnet claiming the exact subnet and gateway of the guest's own uplink: the guest then held 10.88.0.1 as a LOCAL address while its default route pointed at 10.88.0.1 as its GATEWAY, carried two connected routes for the subnet, and suffered intermittent, self-recovering egress blackouts (~24-36 s, roughly hourly) that no host-side check could attribute. The flagship use case funnels agents toward exactly this — working on box, in a box — so the guard, not the operator, has to catch it. setup-host now scans the target subnet BEFORE any mutation: the default route's gateway inside it (the smoking gun — that is this machine's own uplink), or any non-boxnet interface holding an address in it, refuses with the way out named (BOX_SUBNET). A prior boxnet owning the subnet is the one claimant that is NOT a collision — the legitimate re-run — and an existing bridge on a DIFFERENT subnet refuses too: setup-host converges a bridge, it never re-addresses one under live leases. BOX_SUBNET= (validated at the gate, alongside BOX_DNS) moves the whole stack from one place: · the bridge address derives (ipv4.address=/24) · the ACL's gateway carve-out derives — and the ruleset is now converged via 'network acl edit', not created once: the #80 escape hatch (moving the bridge) left the old /32 behind, stranding box DNS inside the 10.0.0.0/8 drop, which presents as a dead resolver, never as a stale ACL · box-firewall reads the gateway off the live bridge ('|| true' because under pipefail an absent bridge would kill the script instead of answering "no bridge yet") · the drill, multiuser rehearsal and migrate-host probes derive the prefix from 'incus network get boxnet ipv4.address' instead of hardcoding 10.88 — a BOX_SUBNET host must not fail its own rehearsals Co-Authored-By: Claude Fable 5 --- drill/drill.sh | 22 +++++--- drill/multiuser.sh | 13 +++-- host/box-firewall.sh | 10 +++- host/migrate-host.sh | 14 +++-- host/setup-host.sh | 131 ++++++++++++++++++++++++++++++++++++++----- 5 files changed, 157 insertions(+), 33 deletions(-) diff --git a/drill/drill.sh b/drill/drill.sh index 257e715..c40d8be 100755 --- a/drill/drill.sh +++ b/drill/drill.sh @@ -94,14 +94,17 @@ in_box() { # · and the interface is NOT called eth0. The PROFILE names the device eth0, # but inside a VM guest predictable naming renames it enp5s0. Six runs of # A3 "not probed" were this, not the network. -# So: read it from inside the box, and select by SUBNET (10.88.x, what boxnet -# hands out) rather than by interface name — docker0 (172.17.x) is the decoy, -# and the NIC's name is the guest's business, not ours. +# So: read it from inside the box, and select by SUBNET (what boxnet hands +# out — read off the network, never hardcoded: BOX_SUBNET moves it, #80) +# rather than by interface name — docker0 (172.17.x) is the decoy, and the +# NIC's name is the guest's business, not ours. +boxnet_gw() { incus network get boxnet ipv4.address 2>/dev/null | cut -d/ -f1; } boxnet_ip() { - local b="$1" ip _i + local b="$1" ip _i pfx + pfx="$(boxnet_gw)"; pfx="${pfx%.*}." for _i in $(seq 1 15); do ip="$(in_box "$b" ip -4 -o addr show scope global \ - | awk '{ for (i = 1; i < NF; i++) if ($i == "inet" && $(i+1) ~ /^10\.88\./) { split($(i+1), a, "/"); print a[1]; exit } }')" + | awk -v p="$pfx" '{ for (i = 1; i < NF; i++) if ($i == "inet" && index($(i+1), p) == 1) { split($(i+1), a, "/"); print a[1]; exit } }')" [ -n "$ip" ] && { printf '%s\n' "$ip"; return 0; } sleep 2 done @@ -693,10 +696,11 @@ fi # nothing serves and read refused-vs-dropped — refused would mean the box's # packet reached the host's stack, which is the thing the firewall must prevent. # (No background listener: one less process to leak, one less way to wedge.) -hv="$(box_probe archive http://10.88.0.1:8099)" +gw="$(boxnet_gw)" +hv="$(box_probe archive "http://$gw:8099")" case "$hv" in reachable|refused) - no "THE BOX'S PACKETS REACH THE HOST on 10.88.0.1:8099 [$hv] — the firewall rules are not holding" + no "THE BOX'S PACKETS REACH THE HOST on $gw:8099 [$hv] — the firewall rules are not holding" aud "A2 box→host: FAIL — $hv (the packet reached the host's stack)" ;; dropped) ok "box → host is blocked (no path to the machine's sockets)" @@ -747,7 +751,7 @@ elif [ -n "$PEER_IP" ]; then fi else no "could not read peer's boxnet address — the sibling probe never ran" - aud "A3 sibling: NOT PROBED (no 10.88.x address on peer)" + aud "A3 sibling: NOT PROBED (no boxnet address on peer)" fi # C5 — DNS enumeration (#15 A4). Now a CONTRACT, not an observation: setup-host @@ -936,7 +940,7 @@ else || no "migrate: legacy box is NOT on box-net" lip="$(boxnet_ip legacybox)" [ -n "$lip" ] && ok "migrate: legacy box got a boxnet address ($lip) — network move landed" \ - || no "migrate: legacy box has no 10.88 address — the move did not take" + || no "migrate: legacy box has no boxnet address — the move did not take" in_box legacybox getent hosts deb.debian.org >/dev/null 2>&1 \ && ok "migrate: re-homed box resolves + reaches the internet on its new leg" \ || no "migrate: re-homed box cannot resolve on boxnet" diff --git a/drill/multiuser.sh b/drill/multiuser.sh index a49fe5d..9de9616 100644 --- a/drill/multiuser.sh +++ b/drill/multiuser.sh @@ -105,6 +105,11 @@ probe_up() { # probe_up echo "$r" } +# The hardened network's gateway and prefix, read off the network — never +# hardcoded, because BOX_SUBNET moves the whole subnet now (#80). +boxnet_gw() { incus network get boxnet ipv4.address 2>/dev/null | cut -d/ -f1; } +boxnet_pfx() { local gw; gw="$(boxnet_gw)"; printf '%s.' "${gw%.*}"; } + cleanup() { [ "$KEEP" = 1 ] && { echo "(--keep: users and boxes left for inspection)"; return; } echo @@ -194,8 +199,8 @@ fi rm -f "$mintlog" as_u "$U1" box list 2>/dev/null | grep -q '^mine ' && ok "(b) box list shows mine" || no "(b) box list does not show mine" as_u "$U1" box exec mine -- true >/dev/null 2>&1 && ok "(b) box exec mine -- true" || no "(b) box exec failed" -as_u "$U1" box info mine 2>/dev/null | grep -q '10\.88\.' \ - && ok "(g) box info shows a boxnet (10.88.x) address — placed on the hardened network" \ +as_u "$U1" box info mine 2>/dev/null | grep -qF "$(boxnet_pfx)" \ + && ok "(g) box info shows a boxnet ($(boxnet_pfx)x) address — placed on the hardened network" \ || no "(g) mine has no boxnet address in box info" as_u "$U1" box snapshot mine s1 >/dev/null 2>&1 && ok "(b) box snapshot mine s1" || no "(b) snapshot refused" as_u "$U1" box restore mine s1 >/dev/null 2>&1 && ok "(b) box restore mine s1 (the incus 6 'snapshot restore' spelling)" || no "(b) restore failed" @@ -224,14 +229,14 @@ phase "g. the isolation contract, measured from INSIDE the boxes" ip1="$(incus --project "$p1" list mine --format csv --columns 4 2>/dev/null | tr -d '"' | sed 's/ (.*//' | head -n1)" ip2="$(incus --project "$p2" list mine --format csv --columns 4 2>/dev/null | tr -d '"' | sed 's/ (.*//' | head -n1)" inf "$U1's mine: ${ip1:-} $U2's mine: ${ip2:-}" -case "$ip1" in 10.88.*) ok "(g) $U1's box holds a boxnet lease" ;; *) no "(g) $U1's box is NOT on boxnet: '$ip1'" ;; esac +case "$ip1" in "$(boxnet_pfx)"*) ok "(g) $U1's box holds a boxnet lease" ;; *) no "(g) $U1's box is NOT on boxnet: '$ip1'" ;; esac r="$(probe_up "$U1" mine https://1.1.1.1)" [ "$r" = reachable ] && ok "(g) egress to the public internet works (curl 1.1.1.1: $r)" || no "(g) public egress broken: $r" as_u "$U1" timeout -k 5 20 incus exec mine -- getent hosts deb.debian.org >/dev/null 2>&1 \ && ok "(g) public DNS resolves (via the pinned resolver)" || no "(g) DNS broken inside the box" -r="$(probe_from "$U1" mine "http://10.88.0.1:22")" +r="$(probe_from "$U1" mine "http://$(boxnet_gw):22")" [ "$r" = dropped ] && ok "(g) box → host is dropped (gateway :22: $r)" || no "(g) box can reach the HOST: $r" r="$(probe_from "$U1" mine "http://192.168.0.1")" [ "$r" = dropped ] && ok "(g) box → RFC1918 is dropped ($r)" || no "(g) box reaches private space: $r" diff --git a/host/box-firewall.sh b/host/box-firewall.sh index edd8374..bb96b50 100644 --- a/host/box-firewall.sh +++ b/host/box-firewall.sh @@ -5,8 +5,16 @@ # Docker's DOCKER-USER rules are runtime-only and need re-applying). set -euo pipefail -GW=10.88.0.1 NET=boxnet +# The gateway is read off the live bridge, not hardcoded: the subnet is an +# input now (BOX_SUBNET, setup-host.sh — #80), and a bridge moved off a +# colliding subnet must keep its firewall. setup-host runs us after the +# bridge exists, so the live read is the truth at install time; UFW rules +# persist across boots on their own, so the default only papers over the +# no-bridge-yet window at boot on a default-subnet host. +# ('|| true': under pipefail an absent bridge would kill the script here.) +GW="$(ip -4 -o addr show dev "$NET" 2>/dev/null | awk '{ split($4, a, "/"); print a[1]; exit }' || true)" +[ -n "$GW" ] || GW=10.88.0.1 if command -v ufw >/dev/null && ufw status 2>/dev/null | grep -q "Status: active"; then if ! ufw status | grep "on $NET" | grep -q "DENY"; then diff --git a/host/migrate-host.sh b/host/migrate-host.sh index e7bf238..19a2dde 100644 --- a/host/migrate-host.sh +++ b/host/migrate-host.sh @@ -3,7 +3,7 @@ # # The zero-ceremony transition is just install.sh + setup-host.sh: that leaves # a DUAL-STACK host where legacy boxes (tag user.claudebox=1, claudenet/10.87, -# claude-dev) keep working while new mints land on boxnet/10.88. This script is +# claude-dev) keep working while new mints land on boxnet. This script is # the two things that path does not do: # # migrate-host.sh --box re-home ONE legacy box onto the new stack @@ -88,17 +88,19 @@ rehome_one() { incus start "$b" >/dev/null 2>&1 || { warn "$b: did not restart — start it by hand"; return 1; } # 3. VERIFY THE EFFECT, not the exit codes (the whole repo's lesson). The box - # must be on 10.88 and actually resolve+reach the internet on its new leg - # before we call it migrated. - local _i ip + # must be on boxnet's subnet (read off the network — BOX_SUBNET moves it, + # #80) and actually resolve+reach the internet on its new leg before we + # call it migrated. + local _i ip pfx + pfx="$(incus network get boxnet ipv4.address 2>/dev/null | cut -d/ -f1)"; pfx="${pfx%.*}." ip="" for _i in $(seq 1 30); do ip="$(incus exec "$b" -- ip -4 -o addr show scope global /dev/null \ - | awk '{for(i=1;i/dev/null 2>&1; then # LAST, and only once the move is VERIFIED: drop the legacy tag. Until this # point the box wears both tags, so a failure anywhere above leaves it a diff --git a/host/setup-host.sh b/host/setup-host.sh index dff7171..ea0d636 100755 --- a/host/setup-host.sh +++ b/host/setup-host.sh @@ -45,6 +45,88 @@ else exit 1 fi +# --- The subnet, and the refusal to build on one something already owns ----- +# (#80.) The stack's subnet was hardcoded, and running setup-host INSIDE a box +# gave the guest a nested boxnet claiming the exact subnet and gateway of its +# own uplink: the guest then held its gateway's address as a LOCAL address, +# carried two connected routes for the subnet, and suffered intermittent, +# self-recovering egress blackouts nobody could attribute — the host looked +# clean the whole time. The flagship use case funnels agents toward doing +# exactly this (working on box, in a box), so the guard must refuse BEFORE +# any mutation, and name the way out (BOX_SUBNET). + +# BOX_SUBNET must be a /24 with a zero host octet — a.b.c.0/24. Everything +# the stack derives (the bridge address, the gateway carve-out, the firewall) +# assumes that shape, and a garbage value must die HERE, never inside an +# incus create or an nft rule. +valid_subnet() { + local o a="" b="" c="" rest="" + case "$1" in *.0/24) ;; *) return 1 ;; esac + IFS=. read -r a b c rest <<<"${1%/24}" + [ "$rest" = 0 ] || return 1 + for o in "$a" "$b" "$c"; do + case "$o" in ''|*[!0-9]*) return 1 ;; esac + [ "${#o}" -le 3 ] && [ "$o" -le 255 ] || return 1 + done +} + +# Who, other than box's own bridge, already owns an address inside $1? +# Prints the claimant and succeeds when the subnet is claimed by a FOREIGNER; +# stays silent and fails when it is free — or held only by boxnet, which is +# the legitimate re-run, converging a stack this script built before. The +# most telling claimant is the default route's gateway: if it sits inside the +# target subnet, this machine's own uplink lives there — i.e. this is almost +# certainly the inside of a box. Pure over `ip` output, so test/cli.sh can +# drive it against canned tables with a shim ip. +subnet_claimant() { + local pfx hit + pfx="${1%0/24}" + hit="$(ip -4 route show default 2>/dev/null | awk -v p="$pfx" ' + { gw = ""; dev = "" + for (i = 1; i < NF; i++) { if ($i == "via") gw = $(i+1); if ($i == "dev") dev = $(i+1) } + if (index(gw, p) == 1 && dev != "boxnet") { + print "this machine\047s own DEFAULT GATEWAY (" gw " via " dev ")"; exit } }')" + if [ -z "$hit" ]; then + hit="$(ip -4 -o addr show 2>/dev/null | awk -v p="$pfx" ' + $2 != "boxnet" && index($4, p) == 1 { print "interface " $2 " (" $4 ")"; exit }')" + fi + [ -n "$hit" ] && printf '%s\n' "$hit" +} + +BOX_SUBNET="${BOX_SUBNET:-10.88.0.0/24}" +if ! valid_subnet "$BOX_SUBNET"; then + echo "ERROR: BOX_SUBNET='$BOX_SUBNET' is not a sane subnet — the stack takes a" >&2 + echo " /24 with a zero host octet, e.g. BOX_SUBNET=10.89.0.0/24" >&2 + exit 1 +fi +BOX_GW="${BOX_SUBNET%.0/24}.1" + +if hit="$(subnet_claimant "$BOX_SUBNET")"; then + echo "ERROR: refusing to build boxnet on $BOX_SUBNET — that subnet is already" >&2 + echo " claimed here by $hit." >&2 + echo " If that is this machine's uplink, you are INSIDE a box: a nested" >&2 + echo " stack on the guest's own subnet captures its gateway address and" >&2 + echo " blackholes its egress, intermittently (issue #80)." >&2 + echo " Nothing was changed. To build a nested stack anyway, pick a free" >&2 + echo " subnet: BOX_SUBNET=10.89.0.0/24 box setup-host" >&2 + exit 1 +fi + +# A bridge this script built before is the one claimant that is NOT a +# collision — but it must AGREE with the target: setup-host converges an +# existing bridge, it never re-addresses one (boxes hold leases on it). +# ('|| true': under pipefail, `ip … dev boxnet` on a fresh host — no such +# device — would kill the script right here instead of answering "no bridge".) +have_gw="$(ip -4 -o addr show dev boxnet 2>/dev/null | awk '{ split($4, a, "/"); print a[1]; exit }' || true)" +if [ -n "$have_gw" ] && [ "$have_gw" != "$BOX_GW" ]; then + echo "ERROR: boxnet already exists on ${have_gw%.*}.0/24 and the target is $BOX_SUBNET —" >&2 + echo " setup-host converges an existing bridge, it never re-addresses one." >&2 + echo " Re-run with the bridge's own subnet:" >&2 + echo " BOX_SUBNET=${have_gw%.*}.0/24 box setup-host" >&2 + echo " (or move the bridge first: incus network set boxnet ipv4.address $BOX_GW/24)" >&2 + exit 1 +fi + # apt, unattended-safe. install.sh now runs us without a human watching, and # a fresh cloud image has apt-daily/unattended-upgrades holding the dpkg lock # for the first minutes of its life — plain 'apt-get install' then waits on it @@ -145,22 +227,45 @@ PRESEED fi # Isolated NAT network. IPv6 off: one less egress path to reason about. -# 10.88, not 10.87: a pre-rename host may still carry claudenet on 10.87 with -# legacy boxes attached — two bridges must not claim one subnet. +# The default is 10.88 — not 10.87: a pre-rename host may still carry +# claudenet on 10.87 with legacy boxes attached — two bridges must not claim +# one subnet. BOX_SUBNET (validated and cleared by the #80 guard above) picks +# another /24; the gateway and every rule below derive from it. incus network show boxnet >/dev/null 2>&1 || incus network create boxnet \ - ipv4.address=10.88.0.1/24 ipv4.nat=true ipv6.address=none + ipv4.address="$BOX_GW/24" ipv4.nat=true ipv6.address=none # ACL: default egress allow (internet), explicit drops for private space. -# Gateway carve-out first so instance DNS (dnsmasq on 10.88.0.1) survives. -if ! incus network acl show box-isolate >/dev/null 2>&1; then - incus network acl create box-isolate - incus network acl rule add box-isolate egress action=allow destination=10.88.0.1/32 - incus network acl rule add box-isolate egress action=drop destination=10.0.0.0/8 - incus network acl rule add box-isolate egress action=drop destination=172.16.0.0/12 - incus network acl rule add box-isolate egress action=drop destination=192.168.0.0/16 - incus network acl rule add box-isolate egress action=drop destination=169.254.0.0/16 - incus network acl rule add box-isolate egress action=drop destination=100.64.0.0/10 -fi +# Gateway carve-out first so instance DNS (dnsmasq on the gateway) survives. +# 'edit' the full shipped ruleset, not create-once: the carve-out derives +# from BOX_SUBNET now, and a bridge moved off a colliding subnet (#80's +# escape hatch) left the OLD /32 behind — box DNS to the new gateway then +# died inside the 10.0.0.0/8 drop, looking like a dead resolver, not a stale +# ACL. A conditional 'rule add' cannot converge that (the stale carve-out +# would survive beside the new one); replacing the ruleset does, idempotently. +incus network acl show box-isolate >/dev/null 2>&1 || incus network acl create box-isolate +incus network acl edit box-isolate <