README.md

@@ -38,9 +38,23 @@ rig bootstrap runner --hostname my-ci-box
 
 What it does: installs `curl ca-certificates unattended-upgrades` (and
 enables periodic unattended upgrades); writes an sshd hardening drop-in
-(`PermitRootLogin prohibit-password`, `PasswordAuthentication no`); installs
+(`PermitRootLogin prohibit-password`, `PasswordAuthentication no`) and
+**verifies it took effect** via `sshd -T`; sets the system hostname; installs
 tailscale and joins your tailnet.
 
+> **Why the drop-in is `00-rig.conf` and not `99-`.** `sshd_config` is
+> **first-wins** — *"for each keyword, the first obtained value will be used"*
+> (`sshd_config(5)`) — and `Include` expands its glob in lexical order. Cloud
+> images ship `/etc/ssh/sshd_config.d/50-cloud-init.conf` carrying
+> `PasswordAuthentication yes`, so a `99-` drop-in is read **second** and every
+> keyword in it is silently discarded. This is the opposite of the
+> last-wins convention most config systems use, and it shipped green here for
+> a month: rig asserted the *file existed* rather than what `sshd` actually
+> resolved, and the Incus rehearsal container has no cloud-init drop-in to
+> lose to. Every Hetzner box rig had bootstrapped was still serving
+> `passwordauthentication yes`. `bootstrap` now sweeps a stale `99-rig.conf`
+> on re-run, and refuses to claim success unless `sshd -T` agrees.
+
 **The pre-auth key:** provide it via the `TS_AUTHKEY` env var or type it at
 the interactive prompt. Use a **single-use, tagged, short-expiry** key. It
 lives in process memory only — rig never writes a credential to disk.

commands/bootstrap.sh

@@ -11,7 +11,7 @@ usage() {
   cat <<'EOF'
 usage: rig bootstrap <control-plane|workload|runner> [--hostname <name>] [--ts-tag <tag>]
 
-  --hostname  tailnet hostname (default: the role name)
+  --hostname  system + tailnet hostname (default: the role name)
   --ts-tag    tailnet tag to advertise (default: tag:server;
               role runner defaults to tag:ci and refuses tag:server —
               a CI box executes repo-controlled code, and your server
@@ -68,12 +68,12 @@ else
   warn "cannot read /etc/os-release; proceeding anyway"
 fi
 
-# --- pre-auth key (env override, else prompt; never touches disk) ------------
+# The pre-auth key is acquired LATER, in the tailscale block — and only if the
-if [ -z "${TS_AUTHKEY:-}" ]; then
+# box has not already joined. rig is convergent by contract, so re-running it to
-  read -rsp "tailscale pre-auth key (single-use, tagged, <=1h expiry): " TS_AUTHKEY
+# pick up a fix (e.g. the 2026-07-12 sshd first-wins fix) must not demand a
-  echo
+# credential it will never spend: prompting up front made the repair path cost a
-fi
+# throwaway Tailscale key, which is exactly the friction that stops people from
-[ -n "$TS_AUTHKEY" ] || die "empty pre-auth key"
+# re-running it.
 
 # --- packages ----------------------------------------------------------------
 export DEBIAN_FRONTEND=noninteractive
@@ -92,14 +92,39 @@ APT::Periodic::Unattended-Upgrade "1";
 EOF
 
 # --- sshd hardening (restart only when the drop-in actually changed) ---------
-DROPIN=/etc/ssh/sshd_config.d/99-rig.conf
+# The name must sort BEFORE cloud-init's drop-in. sshd_config is FIRST-wins
+# ("for each keyword, the first obtained value will be used" — sshd_config(5)),
+# and Include expands the glob in lexical order. Cloud images ship
+# /etc/ssh/sshd_config.d/50-cloud-init.conf carrying `PasswordAuthentication
+# yes`, so the old 99-rig.conf was read second and silently lost every keyword
+# it set. 00- wins. (Found 2026-07-12: every Hetzner box rig had bootstrapped
+# was still serving `passwordauthentication yes`. The Incus rehearsal never
+# caught it — a pristine Debian container has no cloud-init drop-in.)
+DROPIN=/etc/ssh/sshd_config.d/00-rig.conf
+LEGACY_DROPIN=/etc/ssh/sshd_config.d/99-rig.conf
 TMP="$(mktemp)"
 cat > "$TMP" <<'EOF'
 PermitRootLogin prohibit-password
 PasswordAuthentication no
 EOF
-if ! cmp -s "$TMP" "$DROPIN" 2>/dev/null; then
+if ! cmp -s "$TMP" "$DROPIN" 2>/dev/null || [ -e "$LEGACY_DROPIN" ]; then
+  BACKUP=""
+  [ -e "$DROPIN" ] && { BACKUP="$(mktemp)"; cp -a "$DROPIN" "$BACKUP"; }
   install -m 0644 "$TMP" "$DROPIN"
+  rm -f "$LEGACY_DROPIN"   # sweep the losing file from already-bootstrapped boxes
+
+  # Validate the MERGED config BEFORE bouncing the daemon. On a box whose only
+  # door is SSH, `systemctl restart ssh` against a config sshd refuses to parse
+  # leaves no listener and no way back in. `sshd -t` parses everything sshd
+  # would parse — our drop-in, cloud-init's, and any third-party file — so a
+  # broken neighbour is caught here rather than after the door has shut.
+  if ! sshd -t 2>/dev/null; then
+    if [ -n "$BACKUP" ]; then cp -a "$BACKUP" "$DROPIN"; else rm -f "$DROPIN"; fi
+    rm -f "$TMP" "$BACKUP"
+    die "sshd rejects the merged config; drop-in rolled back, daemon untouched. Run 'sshd -t' to see which file is bad."
+  fi
+  rm -f "$BACKUP"
+
   systemctl restart ssh
   log "sshd hardening drop-in installed"
 else
@@ -107,14 +132,49 @@ else
 fi
 rm -f "$TMP"
 
+# Assert the EFFECTIVE config, not the file's existence — asserting the file is
+# what let the first-wins bug ship green. `sshd -T` is what the daemon actually
+# resolved, cloud-init and all.
+eff="$(sshd -T 2>/dev/null)" || die "sshd -T failed; refusing to claim a hardened box"
+echo "$eff" | grep -qx 'passwordauthentication no' \
+  || die "sshd still resolves passwordauthentication=yes — a drop-in is beating ${DROPIN}; check ls /etc/ssh/sshd_config.d/"
+echo "$eff" | grep -qxE 'permitrootlogin (prohibit-password|without-password)' \
+  || die "sshd still permits root password login — check ls /etc/ssh/sshd_config.d/"
+log "sshd hardening verified (sshd -T: passwordauthentication no)"
+
+# --- system hostname ----------------------------------------------------------
+# Set the SYSTEM hostname too, not just the tailnet one. Until 2026-07-12 rig
+# passed --hostname only to `tailscale up`, so a box reached as `coolify-box`
+# still greeted the operator with Hetzner's default (`root@internal-tooling`).
+# The shell prompt is the operator's only "am I on the right box" signal before
+# they run something destructive, and it was lying on every box rig built.
+if [ "$(hostname)" != "$TS_HOSTNAME" ]; then
+  log "setting system hostname to ${TS_HOSTNAME}"
+  hostnamectl set-hostname "$TS_HOSTNAME"
+  # keep 127.0.1.1 in step, or sudo/sshd warn about an unresolvable host
+  if grep -qE '^127\.0\.1\.1[[:space:]]' /etc/hosts; then
+    sed -i -E "s/^127\.0\.1\.1[[:space:]].*/127.0.1.1\t${TS_HOSTNAME}/" /etc/hosts
+  else
+    printf '127.0.1.1\t%s\n' "$TS_HOSTNAME" >> /etc/hosts
+  fi
+else
+  log "system hostname already ${TS_HOSTNAME}"
+fi
+
 # --- tailscale ----------------------------------------------------------------
 if ! command -v tailscale >/dev/null 2>&1; then
   log "installing tailscale"
   curl -fsSL https://tailscale.com/install.sh | sh
 fi
 if tailscale status >/dev/null 2>&1; then
-  log "tailnet already joined; skipping tailscale up"
+  log "tailnet already joined; skipping tailscale up (no pre-auth key needed)"
 else
+  # env override, else prompt; never touches disk
+  if [ -z "${TS_AUTHKEY:-}" ]; then
+    read -rsp "tailscale pre-auth key (single-use, tagged, <=1h expiry): " TS_AUTHKEY
+    echo
+  fi
+  [ -n "${TS_AUTHKEY:-}" ] || die "empty pre-auth key"
   log "joining tailnet as ${TS_HOSTNAME} (${TS_TAG})"
   tailscale up --authkey="$TS_AUTHKEY" --hostname="$TS_HOSTNAME" --advertise-tags="$TS_TAG"
 fi