From 77974d99d6d014fedbadfb78d79b24befc9bea3f Mon Sep 17 00:00:00 2001 From: claude-hdb Date: Sun, 12 Jul 2026 15:26:34 +0000 Subject: [PATCH] fix(bootstrap): sshd hardening never applied on cloud images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rig wrote its hardening drop-in as 99-rig.conf. Cloud images ship 50-cloud-init.conf carrying `PasswordAuthentication yes`, and sshd_config is FIRST-wins ("for each keyword, the first obtained value will be used" — sshd_config(5)) with Include expanding its glob in lexical order. So 50- was read before 99- and every keyword rig set was silently discarded. Every Hetzner box rig has bootstrapped was still serving `passwordauthentication yes` — confirmed today on coolify-box (CX23) and ci-runner (CX43) by `sshd -T`, and from off-tailnet by `ssh -o PreferredAuthentications=none`. Root logins were never exposed (PermitRootLogin resolved to prohibit-password via Debian's stock config), but any password-bearing non-root account was reachable on a public port 22. Three fixes: 1. Name the drop-in 00-rig.conf so it is read first and actually wins; sweep a stale 99-rig.conf on re-run so existing boxes converge. 2. Assert the EFFECTIVE config with `sshd -T` and refuse to claim a hardened box otherwise. Asserting that the file existed is what let this ship green — the Incus rehearsal has no cloud-init drop-in to lose to, so the bug was invisible to the one gate that could have caught it. 3. Set the system hostname, not just the tailnet one. A box reached as `coolify-box` greeted the operator as `root@internal-tooling`; the shell prompt is the only "am I on the right box" signal before a destructive command, and it was lying on every box rig built. Also defer the pre-auth key prompt to the join path. rig is convergent by contract, but re-running it to pick up this fix demanded a Tailscale key it would never spend — friction on precisely the repair path that matters. Co-Authored-By: Claude Opus 4.8 --- README.md | 16 ++++++++++- commands/bootstrap.sh | 65 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 70 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 1e85868..d3eb3ed 100644 --- a/README.md +++ b/README.md @@ -38,9 +38,23 @@ rig bootstrap runner --hostname my-ci-box What it does: installs `curl ca-certificates unattended-upgrades` (and enables periodic unattended upgrades); writes an sshd hardening drop-in -(`PermitRootLogin prohibit-password`, `PasswordAuthentication no`); installs +(`PermitRootLogin prohibit-password`, `PasswordAuthentication no`) and +**verifies it took effect** via `sshd -T`; sets the system hostname; installs tailscale and joins your tailnet. +> **Why the drop-in is `00-rig.conf` and not `99-`.** `sshd_config` is +> **first-wins** — *"for each keyword, the first obtained value will be used"* +> (`sshd_config(5)`) — and `Include` expands its glob in lexical order. Cloud +> images ship `/etc/ssh/sshd_config.d/50-cloud-init.conf` carrying +> `PasswordAuthentication yes`, so a `99-` drop-in is read **second** and every +> keyword in it is silently discarded. This is the opposite of the +> last-wins convention most config systems use, and it shipped green here for +> a month: rig asserted the *file existed* rather than what `sshd` actually +> resolved, and the Incus rehearsal container has no cloud-init drop-in to +> lose to. Every Hetzner box rig had bootstrapped was still serving +> `passwordauthentication yes`. `bootstrap` now sweeps a stale `99-rig.conf` +> on re-run, and refuses to claim success unless `sshd -T` agrees. + **The pre-auth key:** provide it via the `TS_AUTHKEY` env var or type it at the interactive prompt. Use a **single-use, tagged, short-expiry** key. It lives in process memory only — rig never writes a credential to disk. diff --git a/commands/bootstrap.sh b/commands/bootstrap.sh index ac54a9a..af8373a 100755 --- a/commands/bootstrap.sh +++ b/commands/bootstrap.sh @@ -11,7 +11,7 @@ usage() { cat <<'EOF' usage: rig bootstrap [--hostname ] [--ts-tag ] - --hostname tailnet hostname (default: the role name) + --hostname system + tailnet hostname (default: the role name) --ts-tag tailnet tag to advertise (default: tag:server; role runner defaults to tag:ci and refuses tag:server — a CI box executes repo-controlled code, and your server @@ -68,12 +68,12 @@ else warn "cannot read /etc/os-release; proceeding anyway" fi -# --- pre-auth key (env override, else prompt; never touches disk) ------------ -if [ -z "${TS_AUTHKEY:-}" ]; then - read -rsp "tailscale pre-auth key (single-use, tagged, <=1h expiry): " TS_AUTHKEY - echo -fi -[ -n "$TS_AUTHKEY" ] || die "empty pre-auth key" +# The pre-auth key is acquired LATER, in the tailscale block — and only if the +# box has not already joined. rig is convergent by contract, so re-running it to +# pick up a fix (e.g. the 2026-07-12 sshd first-wins fix) must not demand a +# credential it will never spend: prompting up front made the repair path cost a +# throwaway Tailscale key, which is exactly the friction that stops people from +# re-running it. # --- packages ---------------------------------------------------------------- export DEBIAN_FRONTEND=noninteractive @@ -92,14 +92,24 @@ APT::Periodic::Unattended-Upgrade "1"; EOF # --- sshd hardening (restart only when the drop-in actually changed) --------- -DROPIN=/etc/ssh/sshd_config.d/99-rig.conf +# The name must sort BEFORE cloud-init's drop-in. sshd_config is FIRST-wins +# ("for each keyword, the first obtained value will be used" — sshd_config(5)), +# and Include expands the glob in lexical order. Cloud images ship +# /etc/ssh/sshd_config.d/50-cloud-init.conf carrying `PasswordAuthentication +# yes`, so the old 99-rig.conf was read second and silently lost every keyword +# it set. 00- wins. (Found 2026-07-12: every Hetzner box rig had bootstrapped +# was still serving `passwordauthentication yes`. The Incus rehearsal never +# caught it — a pristine Debian container has no cloud-init drop-in.) +DROPIN=/etc/ssh/sshd_config.d/00-rig.conf +LEGACY_DROPIN=/etc/ssh/sshd_config.d/99-rig.conf TMP="$(mktemp)" cat > "$TMP" <<'EOF' PermitRootLogin prohibit-password PasswordAuthentication no EOF -if ! cmp -s "$TMP" "$DROPIN" 2>/dev/null; then +if ! cmp -s "$TMP" "$DROPIN" 2>/dev/null || [ -e "$LEGACY_DROPIN" ]; then install -m 0644 "$TMP" "$DROPIN" + rm -f "$LEGACY_DROPIN" # sweep the losing file from already-bootstrapped boxes systemctl restart ssh log "sshd hardening drop-in installed" else @@ -107,14 +117,49 @@ else fi rm -f "$TMP" +# Assert the EFFECTIVE config, not the file's existence — asserting the file is +# what let the first-wins bug ship green. `sshd -T` is what the daemon actually +# resolved, cloud-init and all. +eff="$(sshd -T 2>/dev/null)" || die "sshd -T failed; refusing to claim a hardened box" +echo "$eff" | grep -qx 'passwordauthentication no' \ + || die "sshd still resolves passwordauthentication=yes — a drop-in is beating ${DROPIN}; check ls /etc/ssh/sshd_config.d/" +echo "$eff" | grep -qxE 'permitrootlogin (prohibit-password|without-password)' \ + || die "sshd still permits root password login — check ls /etc/ssh/sshd_config.d/" +log "sshd hardening verified (sshd -T: passwordauthentication no)" + +# --- system hostname ---------------------------------------------------------- +# Set the SYSTEM hostname too, not just the tailnet one. Until 2026-07-12 rig +# passed --hostname only to `tailscale up`, so a box reached as `coolify-box` +# still greeted the operator with Hetzner's default (`root@internal-tooling`). +# The shell prompt is the operator's only "am I on the right box" signal before +# they run something destructive, and it was lying on every box rig built. +if [ "$(hostname)" != "$TS_HOSTNAME" ]; then + log "setting system hostname to ${TS_HOSTNAME}" + hostnamectl set-hostname "$TS_HOSTNAME" + # keep 127.0.1.1 in step, or sudo/sshd warn about an unresolvable host + if grep -qE '^127\.0\.1\.1[[:space:]]' /etc/hosts; then + sed -i -E "s/^127\.0\.1\.1[[:space:]].*/127.0.1.1\t${TS_HOSTNAME}/" /etc/hosts + else + printf '127.0.1.1\t%s\n' "$TS_HOSTNAME" >> /etc/hosts + fi +else + log "system hostname already ${TS_HOSTNAME}" +fi + # --- tailscale ---------------------------------------------------------------- if ! command -v tailscale >/dev/null 2>&1; then log "installing tailscale" curl -fsSL https://tailscale.com/install.sh | sh fi if tailscale status >/dev/null 2>&1; then - log "tailnet already joined; skipping tailscale up" + log "tailnet already joined; skipping tailscale up (no pre-auth key needed)" else + # env override, else prompt; never touches disk + if [ -z "${TS_AUTHKEY:-}" ]; then + read -rsp "tailscale pre-auth key (single-use, tagged, <=1h expiry): " TS_AUTHKEY + echo + fi + [ -n "${TS_AUTHKEY:-}" ] || die "empty pre-auth key" log "joining tailnet as ${TS_HOSTNAME} (${TS_TAG})" tailscale up --authkey="$TS_AUTHKEY" --hostname="$TS_HOSTNAME" --advertise-tags="$TS_TAG" fi