diff --git a/README.md b/README.md index 1e85868..d3eb3ed 100644 --- a/README.md +++ b/README.md @@ -38,9 +38,23 @@ rig bootstrap runner --hostname my-ci-box What it does: installs `curl ca-certificates unattended-upgrades` (and enables periodic unattended upgrades); writes an sshd hardening drop-in -(`PermitRootLogin prohibit-password`, `PasswordAuthentication no`); installs +(`PermitRootLogin prohibit-password`, `PasswordAuthentication no`) and +**verifies it took effect** via `sshd -T`; sets the system hostname; installs tailscale and joins your tailnet. +> **Why the drop-in is `00-rig.conf` and not `99-`.** `sshd_config` is +> **first-wins** — *"for each keyword, the first obtained value will be used"* +> (`sshd_config(5)`) — and `Include` expands its glob in lexical order. Cloud +> images ship `/etc/ssh/sshd_config.d/50-cloud-init.conf` carrying +> `PasswordAuthentication yes`, so a `99-` drop-in is read **second** and every +> keyword in it is silently discarded. This is the opposite of the +> last-wins convention most config systems use, and it shipped green here for +> a month: rig asserted the *file existed* rather than what `sshd` actually +> resolved, and the Incus rehearsal container has no cloud-init drop-in to +> lose to. Every Hetzner box rig had bootstrapped was still serving +> `passwordauthentication yes`. `bootstrap` now sweeps a stale `99-rig.conf` +> on re-run, and refuses to claim success unless `sshd -T` agrees. + **The pre-auth key:** provide it via the `TS_AUTHKEY` env var or type it at the interactive prompt. Use a **single-use, tagged, short-expiry** key. It lives in process memory only — rig never writes a credential to disk. diff --git a/commands/bootstrap.sh b/commands/bootstrap.sh index cc0f4a9..babe626 100755 --- a/commands/bootstrap.sh +++ b/commands/bootstrap.sh @@ -11,7 +11,7 @@ usage() { cat <<'EOF' usage: rig bootstrap [--hostname ] [--ts-tag ] - --hostname tailnet hostname (default: the role name) + --hostname system + tailnet hostname (default: the role name) --ts-tag tailnet tag to advertise (default: tag:server; role runner defaults to tag:ci and refuses tag:server — a CI box executes repo-controlled code, and your server @@ -58,24 +58,22 @@ fi # --- guards ------------------------------------------------------------------ [ "$(id -u)" -eq 0 ] || die "must run as root" if [ -r /etc/os-release ]; then - # Sourced in a subshell: os-release defines VERSION, NAME, ID, etc. — - # sourcing it in the main shell silently clobbers same-named script vars. # shellcheck source=/dev/null - OS_FAMILY="$(. /etc/os-release && printf '%s %s' "${ID:-}" "${ID_LIKE:-}")" - case "$OS_FAMILY" in + . /etc/os-release + case "${ID:-} ${ID_LIKE:-}" in *debian*) ;; - *) warn "not a Debian-family system (${OS_FAMILY:-unknown}); proceeding anyway" ;; + *) warn "not a Debian-family system (ID=${ID:-unknown}); proceeding anyway" ;; esac else warn "cannot read /etc/os-release; proceeding anyway" fi -# --- pre-auth key (env override, else prompt; never touches disk) ------------ -if [ -z "${TS_AUTHKEY:-}" ]; then - read -rsp "tailscale pre-auth key (single-use, tagged, <=1h expiry): " TS_AUTHKEY - echo -fi -[ -n "$TS_AUTHKEY" ] || die "empty pre-auth key" +# The pre-auth key is acquired LATER, in the tailscale block — and only if the +# box has not already joined. rig is convergent by contract, so re-running it to +# pick up a fix (e.g. the 2026-07-12 sshd first-wins fix) must not demand a +# credential it will never spend: prompting up front made the repair path cost a +# throwaway Tailscale key, which is exactly the friction that stops people from +# re-running it. # --- packages ---------------------------------------------------------------- export DEBIAN_FRONTEND=noninteractive @@ -94,14 +92,39 @@ APT::Periodic::Unattended-Upgrade "1"; EOF # --- sshd hardening (restart only when the drop-in actually changed) --------- -DROPIN=/etc/ssh/sshd_config.d/99-rig.conf +# The name must sort BEFORE cloud-init's drop-in. sshd_config is FIRST-wins +# ("for each keyword, the first obtained value will be used" — sshd_config(5)), +# and Include expands the glob in lexical order. Cloud images ship +# /etc/ssh/sshd_config.d/50-cloud-init.conf carrying `PasswordAuthentication +# yes`, so the old 99-rig.conf was read second and silently lost every keyword +# it set. 00- wins. (Found 2026-07-12: every Hetzner box rig had bootstrapped +# was still serving `passwordauthentication yes`. The Incus rehearsal never +# caught it — a pristine Debian container has no cloud-init drop-in.) +DROPIN=/etc/ssh/sshd_config.d/00-rig.conf +LEGACY_DROPIN=/etc/ssh/sshd_config.d/99-rig.conf TMP="$(mktemp)" cat > "$TMP" <<'EOF' PermitRootLogin prohibit-password PasswordAuthentication no EOF -if ! cmp -s "$TMP" "$DROPIN" 2>/dev/null; then +if ! cmp -s "$TMP" "$DROPIN" 2>/dev/null || [ -e "$LEGACY_DROPIN" ]; then + BACKUP="" + [ -e "$DROPIN" ] && { BACKUP="$(mktemp)"; cp -a "$DROPIN" "$BACKUP"; } install -m 0644 "$TMP" "$DROPIN" + rm -f "$LEGACY_DROPIN" # sweep the losing file from already-bootstrapped boxes + + # Validate the MERGED config BEFORE bouncing the daemon. On a box whose only + # door is SSH, `systemctl restart ssh` against a config sshd refuses to parse + # leaves no listener and no way back in. `sshd -t` parses everything sshd + # would parse — our drop-in, cloud-init's, and any third-party file — so a + # broken neighbour is caught here rather than after the door has shut. + if ! sshd -t 2>/dev/null; then + if [ -n "$BACKUP" ]; then cp -a "$BACKUP" "$DROPIN"; else rm -f "$DROPIN"; fi + rm -f "$TMP" "$BACKUP" + die "sshd rejects the merged config; drop-in rolled back, daemon untouched. Run 'sshd -t' to see which file is bad." + fi + rm -f "$BACKUP" + systemctl restart ssh log "sshd hardening drop-in installed" else @@ -109,14 +132,49 @@ else fi rm -f "$TMP" +# Assert the EFFECTIVE config, not the file's existence — asserting the file is +# what let the first-wins bug ship green. `sshd -T` is what the daemon actually +# resolved, cloud-init and all. +eff="$(sshd -T 2>/dev/null)" || die "sshd -T failed; refusing to claim a hardened box" +echo "$eff" | grep -qx 'passwordauthentication no' \ + || die "sshd still resolves passwordauthentication=yes — a drop-in is beating ${DROPIN}; check ls /etc/ssh/sshd_config.d/" +echo "$eff" | grep -qxE 'permitrootlogin (prohibit-password|without-password)' \ + || die "sshd still permits root password login — check ls /etc/ssh/sshd_config.d/" +log "sshd hardening verified (sshd -T: passwordauthentication no)" + +# --- system hostname ---------------------------------------------------------- +# Set the SYSTEM hostname too, not just the tailnet one. Until 2026-07-12 rig +# passed --hostname only to `tailscale up`, so a box reached as `coolify-box` +# still greeted the operator with Hetzner's default (`root@internal-tooling`). +# The shell prompt is the operator's only "am I on the right box" signal before +# they run something destructive, and it was lying on every box rig built. +if [ "$(hostname)" != "$TS_HOSTNAME" ]; then + log "setting system hostname to ${TS_HOSTNAME}" + hostnamectl set-hostname "$TS_HOSTNAME" + # keep 127.0.1.1 in step, or sudo/sshd warn about an unresolvable host + if grep -qE '^127\.0\.1\.1[[:space:]]' /etc/hosts; then + sed -i -E "s/^127\.0\.1\.1[[:space:]].*/127.0.1.1\t${TS_HOSTNAME}/" /etc/hosts + else + printf '127.0.1.1\t%s\n' "$TS_HOSTNAME" >> /etc/hosts + fi +else + log "system hostname already ${TS_HOSTNAME}" +fi + # --- tailscale ---------------------------------------------------------------- if ! command -v tailscale >/dev/null 2>&1; then log "installing tailscale" curl -fsSL https://tailscale.com/install.sh | sh fi if tailscale status >/dev/null 2>&1; then - log "tailnet already joined; skipping tailscale up" + log "tailnet already joined; skipping tailscale up (no pre-auth key needed)" else + # env override, else prompt; never touches disk + if [ -z "${TS_AUTHKEY:-}" ]; then + read -rsp "tailscale pre-auth key (single-use, tagged, <=1h expiry): " TS_AUTHKEY + echo + fi + [ -n "${TS_AUTHKEY:-}" ] || die "empty pre-auth key" log "joining tailnet as ${TS_HOSTNAME} (${TS_TAG})" tailscale up --authkey="$TS_AUTHKEY" --hostname="$TS_HOSTNAME" --advertise-tags="$TS_TAG" fi diff --git a/commands/runner-install.sh b/commands/runner-install.sh index a2bf3d0..5941b2a 100755 --- a/commands/runner-install.sh +++ b/commands/runner-install.sh @@ -75,13 +75,11 @@ VERSION="${VERSION#v}" # --- guards ---------------------------------------------------------------- [ "$(id -u)" -eq 0 ] || die "must run as root" if [ -r /etc/os-release ]; then - # Sourced in a subshell: os-release defines VERSION (e.g. "13 (trixie)"), - # which would clobber this script's $VERSION. # shellcheck source=/dev/null - OS_FAMILY="$(. /etc/os-release && printf '%s %s' "${ID:-}" "${ID_LIKE:-}")" - case "$OS_FAMILY" in + . /etc/os-release + case "${ID:-} ${ID_LIKE:-}" in *debian*) ;; - *) warn "not a Debian-family system (${OS_FAMILY:-unknown}); proceeding anyway" ;; + *) warn "not a Debian-family system (ID=${ID:-unknown}); proceeding anyway" ;; esac else warn "cannot read /etc/os-release; proceeding anyway" diff --git a/test/cli.sh b/test/cli.sh index df8fbe8..69492b1 100644 --- a/test/cli.sh +++ b/test/cli.sh @@ -68,12 +68,6 @@ else echo "skip: runner non-root refusal (running as root)" fi -# Regression: /etc/os-release defines VERSION (e.g. "13 (trixie)" on Debian); -# sourcing it in the main shell clobbers a script's $VERSION and splices the -# OS string into download URLs. It must only ever be sourced in a subshell. -check "no main-shell os-release sourcing" 1 "" \ - grep -rnE '^[[:space:]]*\.[[:space:]]+/etc/os-release' "$ROOT/commands" - echo "---" echo "$PASS passed, $FAIL failed" [ "$FAIL" -eq 0 ]