diff --git a/drill/RUNS.md b/drill/RUNS.md index 0aa54a4..a18dc3e 100644 --- a/drill/RUNS.md +++ b/drill/RUNS.md @@ -111,6 +111,16 @@ Read this before adding a probe. Every one of these cost a run. cover this; the process table does. `doctor.sh` now checks it, because two cold mints and an hour went into learning it the other way. +12. **A `curl` exit code cannot tell you whether the packet arrived.** Exit 7 is + "failed to connect", and it means *both* `Connection refused` (a RST came + back — **reachable**) and `Could not connect` / `No route to host` (nothing + came back — **isolated**). Opposite conclusions, one number. The drill + mapped 7 → "it arrived" and reported a **working** boundary as a broken one + for two full runs after the fix had landed, while the kernel had `isolated + on` on the bridge ports the whole time. **Read the message.** A refusal is + instant; an unreachable host burns the timeout. This is the same disease as + every other trap here — trusting a proxy for the fact instead of the fact. + ## Diagnosing a stall **Start here: `bash drill/doctor.sh`** — it answers "what state is this host diff --git a/drill/drill.sh b/drill/drill.sh index 49f14e9..4c5e07f 100755 --- a/drill/drill.sh +++ b/drill/drill.sh @@ -106,29 +106,35 @@ claudenet_ip() { return 1 } -# A probe that must not hang, and whose curl exit code IS the finding. -# 0 = connected → reachable -# 7 = connection REFUSED → the packet ARRIVED and something answered (a RST -# from a closed port). Reachable. Not isolated. -# 28 = timed out → the packet was DROPPED in flight. Isolated. -# That 7-vs-28 split is why no listener is needed to prove reachability — and -# the listener is exactly what kept wedging the run (a backgrounded process in -# an 'incus exec' session holds the session open, whatever you redirect). -# A closed port is a perfectly good target: it answers, or it doesn't. -box_curl() { # box_curl [timeout] - local b="$1" url="$2" t="${3:-5}" +# The probe. Its verdict comes from curl's MESSAGE, never from its exit code. +# +# curl exit 7 is "failed to connect" — and it covers BOTH of these: +# · "Connection refused" → a RST came back. The packet ARRIVED. Reachable. +# · "Could not connect to server" / "No route to host" → nothing came back at +# all. The frame went nowhere. ISOLATED. +# Opposite conclusions, one exit code. The drill mapped 7 → "it arrived" and so +# reported a WORKING boundary as a broken one, run after run, while the kernel +# had 'isolated on' the bridge ports the whole time. A refusal is instant; an +# unreachable host burns the timeout. The words say which; the number cannot. +# +# Never hangs: incus exec directly (no login shell), stdin pinned, output landed +# in a file rather than a pipe, hard kill on timeout. +box_probe() { # box_probe [timeout] → reachable | refused | dropped + local b="$1" url="$2" t="${3:-5}" out rc msg + out="$(mktemp)" timeout -k 5 $((t + 15)) incus exec "$b" -- curl -sS -m "$t" -o /dev/null "$url" \ - >/dev/null 2>&1 /dev/null 2>"$out" → reachable | refused | dropped | odd - case "$1" in - 0) echo reachable ;; - 7) echo refused ;; - 28) echo dropped ;; - *) echo "odd($1)" ;; - esac +box_pings() { # box_pings → 0 if it answers ICMP + timeout -k 5 20 incus exec "$1" -- ping -c1 -W2 "$2" >/dev/null 2>&1 /dev/null 2>&1 /dev/null 2>&1 - hv="$(verdict $?)" + hmsg="$(curl -sS -m 5 -o /dev/null "http://$ARCH_IP:8087" 2>&1)"; hrc=$? + if [ "$hrc" -eq 0 ]; then hv=reachable + elif printf '%s' "$hmsg" | grep -q 'Connection refused'; then hv=refused + else hv=dropped + fi case "$hv" in reachable|refused) no "the HOST's packets REACH the box ($ARCH_IP) — the default ingress drop is not holding [$hv]"