Merge pull request #9 from claude-hdb/feat/coolify-backup-timer

feat(coolify): install the control-plane dump as a systemd timer

README.md

@@ -83,6 +83,47 @@ Control-plane box only. Installs Coolify at exactly the pinned version with
 the platform must never move underneath it on its own. Upgrading is an
 explicit re-run with a new pin. The pin is required; there is no default.
 
+### `rig coolify backup install`
+
+Control-plane box only. Installs a **nightly age-encrypted dump of Coolify's own
+database** as a systemd timer.
+
+```sh
+rig coolify backup install
+rig coolify backup install --schedule '*-*-* 02:30:00 UTC' --pg-container coolify-db
+```
+
+- `--schedule <OnCalendar>` — systemd calendar expression (default: `*-*-* 04:00:00 UTC`)
+- `--pg-container` / `--pg-user` / `--pg-db` — Coolify's postgres (defaults: `coolify-db`,
+  `coolify`, `coolify`)
+
+That database holds the GitHub App private key, every registered server's SSH key,
+and every environment value for every environment the control plane manages. It is
+`pg_dump`ed straight into `age` — encrypted **client-side, on the box** — and only
+then shipped to S3. The bucket is never trusted with plaintext.
+
+It is **forensics, not a restore path.** A lost control plane is rebuilt fresh and
+reconciled from your manifest, never restored from this artifact. Which is exactly
+why the plumbing belongs in rig: there *will* be a next control-plane box, and it
+should be backed up from birth rather than depending on someone remembering a
+runbook step mid-incident.
+
+**rig installs the machinery; you supply the bindings.** rig writes
+`/etc/coolify-dump.env` **empty**, `0600`, and never reads it back — no credential
+ever passes through rig. You fill in the age recipient (a *public* key), the S3
+bucket + endpoint, and the S3 credentials. Until you do, the unit **fails loudly on
+every run**: a silent backup is worse than a missing one.
+
+rig cannot verify that the upload works — that needs your credentials. So prove it
+by hand once, rather than letting the timer discover it at 04:00:
+
+```sh
+systemctl start coolify-dump.service
+journalctl -u coolify-dump.service -n 20 --no-pager
+```
+
+A backup you have never read back is not yet a backup.
+
 ### `rig runner install --repo <owner/repo>`
 
 Runner box only, run after `rig bootstrap runner` (the same two-step rhythm

bin/rig

@@ -15,6 +15,11 @@ commands:
       defaults to tag:ci and refuses tag:server.
   coolify install --version <pin>
       Pinned Coolify install (AUTOUPDATE=false). Control-plane box only.
+  coolify backup install [options]
+      Nightly age-encrypted dump of the control-plane database, as a
+      systemd timer. rig installs the machinery and templates an empty
+      0600 bindings file; you fill in the age recipient and S3 details.
+      Control-plane box only. Run as root.
   runner install --repo <owner/repo> --version <pin> [options]
       GitHub Actions runner as a systemd service under an unprivileged
       user — outbound-only, no Docker. Prompts for the short-lived
@@ -34,12 +39,25 @@ case "$cmd" in
   coolify)
     shift
     sub="${1:-}"
-    if [ "$sub" != "install" ]; then
+    case "$sub" in
-      usage >&2
+      install)
-      exit 2
+        shift
-    fi
+        exec "$ROOT/commands/coolify-install.sh" "$@"
-    shift
+        ;;
-    exec "$ROOT/commands/coolify-install.sh" "$@"
+      backup)
+        shift
+        if [ "${1:-}" != "install" ]; then
+          usage >&2
+          exit 2
+        fi
+        shift
+        exec "$ROOT/commands/coolify-backup-install.sh" "$@"
+        ;;
+      *)
+        usage >&2
+        exit 2
+        ;;
+    esac
     ;;
   runner)
     shift

commands/coolify-backup-install.sh

@@ -0,0 +1,228 @@
+#!/usr/bin/env bash
+# rig coolify backup install — nightly age-encrypted dump of the Coolify
+# control-plane database, as a systemd timer.
+#
+# rig installs the machinery and templates the bindings file. It never writes
+# a credential and never reads the file back. Convergent: safe to re-run; an
+# already-filled bindings file is left untouched.
+set -euo pipefail
+
+log()  { printf 'rig-coolify-backup: %s\n' "$*"; }
+warn() { printf 'rig-coolify-backup: WARNING: %s\n' "$*" >&2; }
+die()  { printf 'rig-coolify-backup: ERROR: %s\n' "$1" >&2; exit "${2:-1}"; }
+
+usage() {
+  cat <<'EOF'
+usage: rig coolify backup install [options]
+
+  --schedule <OnCalendar>  systemd OnCalendar expression
+                           (default: *-*-* 04:00:00 UTC)
+  --pg-container <name>    Coolify's postgres container (default: coolify-db)
+  --pg-user <name>         postgres role to dump as (default: coolify)
+  --pg-db <name>           database to dump (default: coolify)
+
+Installs a nightly dump of the Coolify control-plane database: pg_dump piped
+straight into age (encrypted CLIENT-SIDE, on this box) and shipped to an
+S3-compatible bucket. Control-plane box only. Run as root.
+
+That database holds the GitHub App private key, every registered server's SSH
+key, and every environment value for every environment this control plane
+manages — which is why it is never written to disk or to S3 in the clear.
+
+rig installs: age, awscli, /usr/local/sbin/coolify-dump.sh, a systemd service
++ timer, and an EMPTY 0600 bindings file at /etc/coolify-dump.env.
+
+You supply, by filling that file and nothing else: the age recipient (a PUBLIC
+key), the S3 bucket + endpoint, and the S3 credentials. Until you do, the unit
+fails loudly on every run — the correct failure mode for a backup.
+EOF
+}
+
+# --- args (validated before the root check, so errors stay testable) --------
+SCHEDULE="*-*-* 04:00:00 UTC"
+PG_CONTAINER="coolify-db"
+PG_USER="coolify"
+PG_DB="coolify"
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --schedule)
+      [ $# -ge 2 ] || die "--schedule needs a value" 2
+      SCHEDULE="$2"; shift 2 ;;
+    --pg-container)
+      [ $# -ge 2 ] || die "--pg-container needs a value" 2
+      PG_CONTAINER="$2"; shift 2 ;;
+    --pg-user)
+      [ $# -ge 2 ] || die "--pg-user needs a value" 2
+      PG_USER="$2"; shift 2 ;;
+    --pg-db)
+      [ $# -ge 2 ] || die "--pg-db needs a value" 2
+      PG_DB="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) die "unknown flag: $1" 2 ;;
+  esac
+done
+
+[ -n "$SCHEDULE" ]     || die "--schedule must not be empty" 2
+[ -n "$PG_CONTAINER" ] || die "--pg-container must not be empty" 2
+[ -n "$PG_USER" ]      || die "--pg-user must not be empty" 2
+[ -n "$PG_DB" ]        || die "--pg-db must not be empty" 2
+
+# --- guards ----------------------------------------------------------------
+[ "$(id -u)" -eq 0 ] || die "must run as root"
+if [ -r /etc/os-release ]; then
+  # Sourced in a subshell — /etc/os-release defines VERSION and would clobber
+  # a caller's variables (see test/cli.sh's regression check).
+  # shellcheck source=/dev/null
+  OS_FAMILY="$(. /etc/os-release && printf '%s %s' "${ID:-}" "${ID_LIKE:-}")"
+  case "$OS_FAMILY" in
+    *debian*) ;;
+    *) warn "not a Debian-family system (${OS_FAMILY:-unknown}); proceeding anyway" ;;
+  esac
+else
+  warn "cannot read /etc/os-release; proceeding anyway"
+fi
+command -v docker >/dev/null \
+  || die "docker not found — this is a control-plane box command (run rig coolify install first)"
+command -v systemctl >/dev/null || die "systemd is required"
+
+SCRIPT_PATH="/usr/local/sbin/coolify-dump.sh"
+ENV_FILE="/etc/coolify-dump.env"
+UNIT_DIR="/etc/systemd/system"
+
+# --- packages ---------------------------------------------------------------
+# age encrypts the dump before it leaves the box. awscli is used ONLY as an
+# S3 protocol client (--endpoint-url); no AWS account is involved.
+log "installing age + awscli"
+export DEBIAN_FRONTEND=noninteractive
+apt-get update -qq
+apt-get install -y -qq age awscli >/dev/null
+log "age $(age --version 2>/dev/null || echo '?') · $(aws --version 2>&1 | cut -d' ' -f1)"
+
+# --- the dump script --------------------------------------------------------
+log "writing ${SCRIPT_PATH}"
+cat > "$SCRIPT_PATH" <<'DUMP_SCRIPT'
+#!/usr/bin/env bash
+# Nightly age-encrypted dump of the Coolify control-plane database.
+# Installed by `rig coolify backup install` — edit rig, not this copy.
+#
+# Forensics, not a restore path: a lost control plane is rebuilt fresh and
+# reconciled from its manifest, never restored from this artifact. It exists
+# to answer "what WAS the state" and to recover a credential otherwise lost.
+set -euo pipefail
+
+: "${AGE_RECIPIENT:?not set — fill /etc/coolify-dump.env (age PUBLIC key)}"
+: "${S3_BUCKET:?not set — fill /etc/coolify-dump.env (e.g. s3://backups/coolify-db)}"
+: "${S3_ENDPOINT:?not set — fill /etc/coolify-dump.env (e.g. https://hel1.your-objectstorage.com)}"
+
+PG_CONTAINER="${PG_CONTAINER:-coolify-db}"
+PG_USER="${PG_USER:-coolify}"
+PG_DB="${PG_DB:-coolify}"
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "$WORKDIR"' EXIT
+OUT="${WORKDIR}/coolify-db-$(date -u +%Y%m%dT%H%M%SZ).sql.age"
+
+# `set -o pipefail` (above) is load-bearing: without it a failing pg_dump still
+# exits 0 through the pipe, and age faithfully encrypts the truncated output.
+docker exec "$PG_CONTAINER" pg_dump -U "$PG_USER" "$PG_DB" \
+  | age -r "$AGE_RECIPIENT" -o "$OUT"
+
+# A failed dump piped into age still yields a valid, tiny, encrypted file. That
+# would upload cleanly every night and look exactly like a working backup.
+[ -s "$OUT" ] || { printf 'coolify-dump: refusing to upload an empty artifact\n' >&2; exit 1; }
+
+aws s3 cp "$OUT" "${S3_BUCKET}/" --endpoint-url "$S3_ENDPOINT"
+printf 'coolify-dump: uploaded %s (%s bytes)\n' "$(basename "$OUT")" "$(stat -c %s "$OUT")"
+DUMP_SCRIPT
+chmod 700 "$SCRIPT_PATH"
+
+# --- bindings file (templated empty; NEVER clobbered) ------------------------
+if [ -e "$ENV_FILE" ]; then
+  log "${ENV_FILE} exists — leaving it alone (rig never reads or rewrites it)"
+else
+  log "templating ${ENV_FILE} (empty, 0600)"
+  install -m 0600 /dev/null "$ENV_FILE"
+  cat > "$ENV_FILE" <<'ENV_TEMPLATE'
+# Coolify control-plane dump — bindings.
+#
+# rig installed the machinery; these values are yours. rig wrote this file
+# empty and never reads it back. Nothing here is committed anywhere.
+#
+# AGE_RECIPIENT is a PUBLIC key (age1...) — safe to hold on this box. Its
+# PRIVATE half must never live here: whoever holds it can read every secret
+# this control plane manages. Keep it wherever your strictest key lives.
+AGE_RECIPIENT=
+S3_BUCKET=
+S3_ENDPOINT=
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_DEFAULT_REGION=
+ENV_TEMPLATE
+  chmod 600 "$ENV_FILE"
+fi
+
+# --- systemd service + timer -------------------------------------------------
+log "writing ${UNIT_DIR}/coolify-dump.service"
+cat > "$UNIT_DIR/coolify-dump.service" <<UNIT
+[Unit]
+Description=Age-encrypted dump of the Coolify control-plane database
+Documentation=https://github.com/heavy-duty/rig
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=oneshot
+# Defaults rig owns. Listed BEFORE EnvironmentFile so the operator's file wins
+# on any key it sets, while a file that omits them still gets sane values.
+#
+# aws-cli >= 2.23 turns on new default upload checksums that S3-compatible
+# backends (Hetzner, MinIO, Ceph) reject; \`when_required\` restores the older
+# behavior. Debian 13 ships 2.23. Harmless where the backend does support them.
+Environment=AWS_REQUEST_CHECKSUM_CALCULATION=when_required
+Environment=AWS_RESPONSE_CHECKSUM_VALIDATION=when_required
+Environment=PG_CONTAINER=${PG_CONTAINER}
+Environment=PG_USER=${PG_USER}
+Environment=PG_DB=${PG_DB}
+EnvironmentFile=${ENV_FILE}
+ExecStart=${SCRIPT_PATH}
+UNIT
+
+log "writing ${UNIT_DIR}/coolify-dump.timer (${SCHEDULE})"
+cat > "$UNIT_DIR/coolify-dump.timer" <<UNIT
+[Unit]
+Description=Nightly Coolify control-plane dump
+Documentation=https://github.com/heavy-duty/rig
+
+[Timer]
+OnCalendar=${SCHEDULE}
+# Catch a run missed while the box was down, rather than silently skipping a night.
+Persistent=true
+RandomizedDelaySec=15m
+
+[Install]
+WantedBy=timers.target
+UNIT
+
+systemctl daemon-reload
+systemctl enable --now coolify-dump.timer >/dev/null 2>&1
+log "timer enabled — next run: $(systemctl show -P NextElapseUSecRealtime coolify-dump.timer 2>/dev/null || echo '?')"
+
+# --- what rig deliberately did NOT do ----------------------------------------
+cat <<EOF
+
+rig-coolify-backup: installed. The timer is live but the backup does NOT work yet —
+rig has no credentials and cannot verify an upload. Two steps remain, both yours:
+
+  1. Fill in the bindings:  \$EDITOR ${ENV_FILE}
+     (age recipient = a PUBLIC key; S3 bucket, endpoint, access key, secret, region)
+
+  2. Prove it end to end — do NOT wait for the timer to find out:
+       systemctl start coolify-dump.service
+       journalctl -u coolify-dump.service -n 20 --no-pager
+     then confirm the object really landed and is really age ciphertext:
+       aws s3 ls "\$S3_BUCKET/" --endpoint-url "\$S3_ENDPOINT"
+     A backup you have never read back is not yet a backup.
+
+Until step 1 is done the unit fails loudly on every run. That is deliberate — a
+silent backup is worse than a missing one.
+EOF

commands/coolify-install.sh

@@ -39,4 +39,5 @@ log "installing coolify ${VERSION} (AUTOUPDATE=false)"
 curl -fsSL https://cdn.coollabs.io/coolify/install.sh -o /tmp/coolify-install.sh
 bash /tmp/coolify-install.sh "$VERSION"
 log "coolify ${VERSION} installed with AUTOUPDATE=false"
-log "next: your bootstrap runbook (admin user, API token, GitHub App, S3 destination)"
+log "next: rig coolify backup install  (nightly control-plane dump — do this before the box holds anything)"
+log "then: your bootstrap runbook (admin user, API token, GitHub App, S3 destination)"

test/cli.sh

@@ -54,6 +54,18 @@ else
   echo "skip: coolify non-root refusal (running as root)"
 fi
 
+check "bare coolify backup shows usage, exit 2" 2 "usage:" "$ROOT/bin/rig" coolify backup
+check "coolify backup: bad subcommand exits 2"  2 "usage:" "$ROOT/bin/rig" coolify backup frobnicate
+check "coolify backup: --help exits 0"          0 "usage:" "$ROOT/commands/coolify-backup-install.sh" --help
+check "coolify backup: schedule needs value"    2 "needs a value" "$ROOT/commands/coolify-backup-install.sh" --schedule
+check "coolify backup: pg-container needs value" 2 "needs a value" "$ROOT/commands/coolify-backup-install.sh" --pg-container
+check "coolify backup: unknown flag exits 2"    2 "unknown flag"  "$ROOT/commands/coolify-backup-install.sh" --nope
+if [ "$(id -u)" -ne 0 ]; then
+  check "coolify backup: refuses non-root"      1 "must run as root" "$ROOT/commands/coolify-backup-install.sh"
+else
+  echo "skip: coolify backup non-root refusal (running as root)"
+fi
+
 check "bare runner shows usage, exit 2"  2 "usage:"          "$ROOT/bin/rig" runner
 check "runner: --help exits 0"           0 "usage:"          "$ROOT/commands/runner-install.sh" --help
 check "runner: repo required, exit 2"    2 "--repo"          "$ROOT/commands/runner-install.sh" --version 2.335.1