From 79834369b1bfc6e3f21d71cc6da4d89de5f6bd1d Mon Sep 17 00:00:00 2001 From: claude-hdb Date: Mon, 13 Jul 2026 16:30:04 +0000 Subject: [PATCH 1/3] fix: authenticate clones via gh / token, never fall into git's prompt (#13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolveCheckout shelled out to a bare `git clone` and relied entirely on the ambient credential helper. On a workstation with none configured, git falls through to its interactive username/password prompt — which GitHub no longer accepts — and the resulting error talks about *the repository* rather than about cast's missing credentials. Being logged into `gh` does not help: `gh auth login` alone does not wire git's helper (that is `gh auth setup-git`, a separate act most people never run). Not routable around for prod: resolveCheckout refuses --path with --env prod, so the clone is the only path and its auth is mandatory. cast now resolves credentials itself, in order: `gh` borrowed as a per-invocation credential helper (no mutation of the user's global git config), then GITHUB_TOKEN / GH_TOKEN, then the ambient helper. The token is never embedded in the clone URL or in http.extraheader — both leak it into `ps`, and the latter persists it into the clone's git config. The helper reads it from the environment at run time, so what lands in argv is the literal text `$CAST_GIT_TOKEN`, never its value. GIT_TERMINAL_PROMPT=0 on every path: whichever credential was used, git may never fall through to a prompt it cannot satisfy — it can only hang, or hide the real fault. When there were no credentials at all, cast now says so, and names the fix. Note that the empty `credential.helper=` reset clears URL-scoped helpers (`credential.https://github.com.helper`, what `gh auth setup-git` writes) as well as generic ones — verified against a live private clone, along with all three acceptance criteria. Co-Authored-By: Claude Opus 4.8 --- src/resolve.ts | 158 +++++++++++++++++++++++++++++++++++++++++-- test/resolve.test.ts | 95 +++++++++++++++++++++++++- 2 files changed, 245 insertions(+), 8 deletions(-) diff --git a/src/resolve.ts b/src/resolve.ts index fae4f2d..c76012a 100644 --- a/src/resolve.ts +++ b/src/resolve.ts @@ -6,6 +6,127 @@ import type { Desired } from "./diff.js"; import { type ResolvedEnv, resolveTemplate } from "./envtemplate.js"; import { loadManifest } from "./manifest.js"; +// How cast authenticated (or failed to authenticate) a clone. +// +// gh — `gh` is installed and holds a token; borrowed as a credential +// helper for this invocation only +// token — GITHUB_TOKEN / GH_TOKEN in the environment (the CI path) +// ambient — neither; whatever git's own credential helper does, if anything +export type GitAuth = { + source: "gh" | "token" | "ambient"; + configArgs: string[]; + env: Record; +}; + +// A credential helper reads the token from the ENVIRONMENT at run time. The +// alternatives both leak it: a token in the clone URL shows up in `ps` and in +// git's own error messages, and `http.extraheader` additionally persists into +// the clone's .git/config. What lands in argv here is the literal text +// `$CAST_GIT_TOKEN`, never its value. +const TOKEN_HELPER = + '!f() { test "$1" = get || exit 0; echo username=x-access-token; echo "password=$CAST_GIT_TOKEN"; }; f'; + +// `gh auth login` alone does NOT wire git's credential helper — that is +// `gh auth setup-git`, a separate act most people never run. So being logged +// into `gh` does not make `git clone` work, which is exactly the trap #13 +// fell into. Borrowing gh as a helper for this one invocation closes that gap +// without mutating the operator's global git config. +const GH_HELPER = "!gh auth git-credential"; + +function ghHasToken(): boolean { + try { + // A local keyring/config read, not a network call. We never keep the + // value — the helper re-reads it inside git. + execFileSync("gh", ["auth", "token"], { stdio: "pipe" }); + return true; + } catch { + return false; + } +} + +// Resolve clone credentials INSIDE cast, in a fixed order, rather than leaving +// it to whatever the ambient git config happens to do. `credential.helper=` +// (empty) first RESETS the inherited helper list — otherwise a helper +// configured globally is consulted before ours and silently decides the +// outcome, which is the same "the connection target is implicit in a file's +// contents" problem #14 is about. +export function resolveGitAuth( + env: NodeJS.ProcessEnv = process.env, + hasGh: () => boolean = ghHasToken, +): GitAuth { + if (hasGh()) { + return { + source: "gh", + configArgs: [ + "-c", + "credential.helper=", + "-c", + `credential.helper=${GH_HELPER}`, + ], + env: {}, + }; + } + const token = env.GITHUB_TOKEN || env.GH_TOKEN; + if (token) { + return { + source: "token", + configArgs: [ + "-c", + "credential.helper=", + "-c", + `credential.helper=${TOKEN_HELPER}`, + ], + env: { CAST_GIT_TOKEN: token }, + }; + } + return { source: "ambient", configArgs: [], env: {} }; +} + +// GitHub answers "you cannot see this" with a 404, not a 403 — so a private +// repo you lack access to and a repo that does not exist are the same message +// on the wire. The failure text must not pick one; it has to name both, and +// name the credential cast actually used, or the operator debugs the wrong +// half. (The original bug reported *the repository* when the real fault was +// cast's missing credentials.) +export function cloneFailureMessage( + orgRepo: string, + auth: GitAuth, + stderr: string, +): string { + const detail = stderr.trim(); + const tail = detail + ? ["", "git said:", ...detail.split("\n").map((l) => ` ${l}`)] + : []; + if (auth.source === "ambient") { + return [ + `cannot clone ${orgRepo}: no GitHub credentials.`, + "", + "cast looked for, in order:", + " 1. `gh` — not installed, or not logged in (`gh auth token` failed)", + " 2. GITHUB_TOKEN / GH_TOKEN — not set in the environment", + " 3. git's own credential helper — did not supply credentials either", + "", + "Run `gh auth login`, or set GITHUB_TOKEN. (`gh auth setup-git` also works,", + "but cast borrows `gh` as a credential helper on its own, so logging in is", + "enough — you do not need to change your global git config.)", + ...tail, + ].join("\n"); + } + const used = + auth.source === "gh" + ? "`gh` (borrowed as a credential helper for this clone)" + : "GITHUB_TOKEN / GH_TOKEN from the environment"; + return [ + `cannot clone ${orgRepo}: authenticated with ${used}, and GitHub still refused.`, + "", + "GitHub answers 'you cannot see this' with a 404, so this is one of:", + ` - ${orgRepo} does not exist (check the slug)`, + " - it is private and this credential has no access to it", + " - the credential is expired, or lacks the `repo` scope", + ...tail, + ].join("\n"); +} + export function resolveCheckout( orgRepo: string, opts: { env: string; path?: string }, @@ -17,13 +138,36 @@ export function resolveCheckout( } if (opts.path) return opts.path; const dir = mkdtempSync(join(tmpdir(), "infra-checkout-")); - execFileSync( - "git", - ["clone", "--depth", "1", `https://github.com/${orgRepo}.git`, dir], - { - stdio: "pipe", - }, - ); + const auth = resolveGitAuth(); + try { + execFileSync( + "git", + [ + ...auth.configArgs, + "clone", + "--depth", + "1", + `https://github.com/${orgRepo}.git`, + dir, + ], + { + stdio: "pipe", + env: { + ...process.env, + ...auth.env, + // Belt and braces: whatever credential path we took, git may NEVER + // fall through to its interactive username/password prompt. GitHub + // stopped accepting passwords there years ago, so it cannot succeed + // — it can only hang cast, or (in the original report) hand back an + // error about the repository that hides the real fault. + GIT_TERMINAL_PROMPT: "0", + }, + }, + ); + } catch (err) { + const stderr = String((err as { stderr?: Buffer | string })?.stderr ?? ""); + throw new Error(cloneFailureMessage(orgRepo, auth, stderr)); + } return dir; } diff --git a/test/resolve.test.ts b/test/resolve.test.ts index 2adb7fa..1cb9de5 100644 --- a/test/resolve.test.ts +++ b/test/resolve.test.ts @@ -3,7 +3,12 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; import { computeDiff } from "../src/diff.js"; -import { desiredFromManifest, resolveCheckout } from "../src/resolve.js"; +import { + cloneFailureMessage, + desiredFromManifest, + resolveCheckout, + resolveGitAuth, +} from "../src/resolve.js"; describe("resolveCheckout", () => { it("hard-refuses --path with prod", () => { @@ -21,6 +26,94 @@ describe("resolveCheckout", () => { }); }); +describe("resolveGitAuth", () => { + const noGh = () => false; + const yesGh = () => true; + + it("prefers gh, borrowed as a per-invocation credential helper", () => { + const auth = resolveGitAuth({ GITHUB_TOKEN: "t" }, yesGh); + expect(auth.source).toBe("gh"); + expect(auth.configArgs.join(" ")).toContain("!gh auth git-credential"); + // No token is materialized when gh is driving. + expect(auth.env).toEqual({}); + }); + + it("falls back to GITHUB_TOKEN when gh is absent", () => { + const auth = resolveGitAuth({ GITHUB_TOKEN: "ghp_secret" }, noGh); + expect(auth.source).toBe("token"); + expect(auth.env).toEqual({ CAST_GIT_TOKEN: "ghp_secret" }); + }); + + it("accepts GH_TOKEN as well as GITHUB_TOKEN", () => { + const auth = resolveGitAuth({ GH_TOKEN: "ghp_secret" }, noGh); + expect(auth.source).toBe("token"); + expect(auth.env).toEqual({ CAST_GIT_TOKEN: "ghp_secret" }); + }); + + // The acceptance criterion from #13: "the token never appears in process + // arguments or on disk". The helper string git receives must carry the + // NAME of the variable, never its value — sh expands it inside the helper. + it("never puts the token value in the git argv", () => { + const auth = resolveGitAuth({ GITHUB_TOKEN: "ghp_secret" }, noGh); + const argv = auth.configArgs.join(" "); + expect(argv).not.toContain("ghp_secret"); + expect(argv).toContain("$CAST_GIT_TOKEN"); + }); + + // A helper configured globally would otherwise be consulted first and + // silently decide the outcome, defeating the order cast just established. + it("resets the inherited helper list before installing its own", () => { + for (const auth of [ + resolveGitAuth({}, yesGh), + resolveGitAuth({ GITHUB_TOKEN: "t" }, noGh), + ]) { + expect(auth.configArgs.slice(0, 2)).toEqual(["-c", "credential.helper="]); + } + }); + + it("falls through to the ambient helper when there is nothing else", () => { + expect(resolveGitAuth({}, noGh)).toEqual({ + source: "ambient", + configArgs: [], + env: {}, + }); + }); +}); + +describe("cloneFailureMessage", () => { + const ambient = { source: "ambient" as const, configArgs: [], env: {} }; + const gh = { source: "gh" as const, configArgs: [], env: {} }; + + // The original bug: git's own error talked about THE REPOSITORY when the + // real fault was cast having no credentials at all. + it("blames the missing credentials, not the repo, when there were none", () => { + const msg = cloneFailureMessage("heavy-duty/incubator", ambient, ""); + expect(msg).toMatch(/no GitHub credentials/); + expect(msg).toMatch(/gh auth login/); + expect(msg).toMatch(/GITHUB_TOKEN/); + expect(msg).not.toMatch(/does not exist/); + }); + + // ...and the converse: once cast DID authenticate, the repo really is a + // candidate explanation again, and 404-means-403 has to be spelled out. + it("names both roads when a credential was used and GitHub still refused", () => { + const msg = cloneFailureMessage("heavy-duty/incubator", gh, ""); + expect(msg).toMatch(/gh/); + expect(msg).toMatch(/does not exist/); + expect(msg).toMatch(/private/); + expect(msg).not.toMatch(/no GitHub credentials/); + }); + + it("passes git's own stderr through rather than swallowing it", () => { + const msg = cloneFailureMessage( + "heavy-duty/incubator", + ambient, + "fatal: could not read Username for 'https://github.com'", + ); + expect(msg).toMatch(/could not read Username/); + }); +}); + describe("desiredFromManifest", () => { it("maps manifest + templates to Desired[] with resolved env", () => { const dir = mkdtempSync(join(tmpdir(), "infra-co-")); From e4572614364ffce2ecebf43914a7fcff9a9da851 Mon Sep 17 00:00:00 2001 From: claude-hdb Date: Mon, 13 Jul 2026 16:42:45 +0000 Subject: [PATCH 2/3] feat: select the Coolify instance by name instead of editing .coolify.env (#14) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit loadConfig read exactly one COOLIFY_BASE_URL + COOLIFY_ACCESS_TOKEN from /.coolify.env, with no flag or env override: the connection target was implicit in a file's current contents. Retargeting cast meant hand-editing a live credential file — and putting it back afterwards. The failure mode of getting that wrong is running `apply` against production. That is not hypothetical during the prod migration (incubator D-193): the state repo's .coolify.env holds a write+deploy token for the NEW control plane, while the verification gate needs a --full diff against the legacy, hand-built box still serving live users. - Named instances: /.coolify/.env, each with its own base URL and token. --instance on every verb that reaches Coolify. - environments.yaml may bind one per environment (`instance: prod-cp`), so --env selects the right control plane with no flag and no file edit at all. An explicit --instance still wins, so a one-off read against a legacy box needs no change to that file either. - Refuse, don't guess, on an unknown --instance — naming the instances that do exist, in the same spirit as the absent-target refusal (#12/D-237). Falling back to the default here is exactly how a diff meant for a legacy box gets run against production. - An instance may declare COOLIFY_READ_ONLY=true; apply, smoke and server add then refuse it before their first call, even though the token itself would permit the writes. "I pointed the wrong token at the wrong box" becomes an exit code rather than a live incident. - Every command that reaches a Coolify now SAYS which one, next to the team assert. It is the most consequential input and the least visible one. With no --instance and no binding, behavior is byte-for-byte what it was. The CLI tests spawn cast against stub Coolifys that record what they were asked, so "which instance did it actually talk to" is answered from the wire rather than from cast's own console output. Co-Authored-By: Claude Opus 4.8 --- src/bindings.ts | 9 ++ src/cli.ts | 98 ++++++++++++++---- src/config.ts | 117 ++++++++++++++++++++-- test/cli.test.ts | 239 ++++++++++++++++++++++++++++++++++++++++---- test/config.test.ts | 143 ++++++++++++++++++++++++++ 5 files changed, 563 insertions(+), 43 deletions(-) create mode 100644 test/config.test.ts diff --git a/src/bindings.ts b/src/bindings.ts index cc7c5e1..c66d301 100644 --- a/src/bindings.ts +++ b/src/bindings.ts @@ -38,6 +38,15 @@ const BindingsSchema = z // it per environment keeps each one's expectation explicit and // survives a future split into per-environment tokens. team: TeamSchema, + // The named Coolify instance this environment lives on + // (/.coolify/.env). Optional: with no binding and no + // --instance, cast reads /.coolify.env exactly as it always + // has. Binding it here is what lets `--env prod` select the right + // control plane with no flag and no file edit — the connection + // target stops being implicit in a file's current contents. + // An explicit --instance still wins, so a one-off read against a + // legacy box needs no change to this file either. + instance: z.string().optional(), s3_destination: z.string().optional(), // Var-name patterns this environment refuses outright (see // assertEnvVarPolicy). Operator-owned guard: prod typically bans diff --git a/src/cli.ts b/src/cli.ts index e516aaa..b261da9 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -5,7 +5,12 @@ import { parseArgs } from "node:util"; import { parse as parseYaml } from "yaml"; import { type Executor, applyHostnameOverlay, applyPlan } from "./apply.js"; import { githubAppNameFor, loadBindings } from "./bindings.js"; -import { loadCoolifyEnv } from "./config.js"; +import { + type CoolifyInstance, + assertWritable, + formatInstance, + loadInstance, +} from "./config.js"; import { CoolifyClient, HttpError } from "./coolify.js"; import { type Live, @@ -33,6 +38,13 @@ const USAGE = `usage: cast apply / --env [--path ] [--proj the token belongs to that environment's declared team. \`cast team\` alone (no --env) reports the token's team without needing a binding — use it to fill environments.yaml. + --instance + the Coolify to talk to: /.coolify/.env, instead + of /.coolify.env. Bind one per environment in + environments.yaml (\`instance: \`) and --env selects it + with no flag; an explicit --instance still wins. An instance + may declare COOLIFY_READ_ONLY=true, and then no command that + writes will run against it. --project the Coolify project to act on, when it is not named after the repo (the default). A project built by hand in the UI is called @@ -46,6 +58,28 @@ function stateDirFrom(flag: string | undefined): string { return flag ?? process.env.CAST_STATE ?? "."; } +// Resolve which Coolify to talk to, announce it, and open a client on it. +// +// Precedence: --instance > the environment's `instance:` binding > the +// default .coolify.env. Every command that reaches a live Coolify goes through +// here, so every one of them SAYS which Coolify it is about to touch, right +// next to the team assert. The connection target used to be implicit in +// .coolify.env's current contents — retargeting meant hand-editing a live +// credential file and putting it back afterwards, and the failure mode of +// getting it wrong is running `apply` against production. +function openCoolify( + stateDir: string, + flag: string | undefined, + binding?: { instance?: string }, +): { instance: CoolifyInstance; client: CoolifyClient } { + const instance = loadInstance(stateDir, flag ?? binding?.instance); + console.log(formatInstance(instance)); + return { + instance, + client: new CoolifyClient(instance.baseUrl, instance.token), + }; +} + // Live Coolify objects use their own field vocabulary; computeDiff compares // by the DESIRED vocabulary, so each live resource must be projected onto it // or every run reports spurious drift (breaks idempotency, criterion 2). @@ -306,6 +340,7 @@ async function main(): Promise { path: { type: "string" }, state: { type: "string" }, project: { type: "string" }, + instance: { type: "string" }, "hostname-overlay": { type: "string" }, full: { type: "boolean", default: false }, }, @@ -350,8 +385,12 @@ async function main(): Promise { parseYaml(readFileSync(values["hostname-overlay"], "utf8")), ); } - const { baseUrl, token } = loadCoolifyEnv(join(stateDir, ".coolify.env")); - const client = new CoolifyClient(baseUrl, token); + const { instance, client } = openCoolify( + stateDir, + values.instance, + binding, + ); + if (command === "apply") assertWritable(instance, "apply"); // Fail-closed, before the first live read — not merely before the first // write. A wrong-team token makes fetchLive come back empty (the API // resolves what it cannot see to null), so an unasserted `diff` would @@ -438,6 +477,7 @@ async function main(): Promise { user: { type: "string" }, port: { type: "string" }, state: { type: "string" }, + instance: { type: "string" }, }, }); // --env is required: a server is registered under the token's team and @@ -457,8 +497,12 @@ async function main(): Promise { console.error(`environment ${values.env} not in environments.yaml`); return 2; } - const { baseUrl, token } = loadCoolifyEnv(join(stateDir, ".coolify.env")); - const client = new CoolifyClient(baseUrl, token); + const { instance, client } = openCoolify( + stateDir, + values.instance, + binding, + ); + assertWritable(instance, "server add"); const team = await assertTeam(client, binding.team, values.env); console.log(`team ${formatTeam(team)} ✓`); await serverAdd(client, { @@ -474,7 +518,11 @@ async function main(): Promise { const { values } = parseArgs({ args: rest, allowPositionals: true, - options: { state: { type: "string" }, env: { type: "string" } }, + options: { + state: { type: "string" }, + env: { type: "string" }, + instance: { type: "string" }, + }, }); // smoke writes: it POSTs two env vars onto the live smoke_target app and // deletes them again. That is a mutation, so it takes the assert like any @@ -485,14 +533,18 @@ async function main(): Promise { return 2; } const stateDir = stateDirFrom(values.state); - const { baseUrl, token } = loadCoolifyEnv(join(stateDir, ".coolify.env")); - const client = new CoolifyClient(baseUrl, token); const bindings = loadBindings(join(stateDir, "environments.yaml")); const binding = bindings.environments[values.env]; if (!binding) { console.error(`environment ${values.env} not in environments.yaml`); return 2; } + const { instance, client } = openCoolify( + stateDir, + values.instance, + binding, + ); + assertWritable(instance, "smoke"); const team = await assertTeam(client, binding.team, values.env); console.log(`team ${formatTeam(team)} ✓`); if (!bindings.smoke_target) { @@ -518,11 +570,27 @@ async function main(): Promise { const { values } = parseArgs({ args: rest, allowPositionals: true, - options: { state: { type: "string" }, env: { type: "string" } }, + options: { + state: { type: "string" }, + env: { type: "string" }, + instance: { type: "string" }, + }, }); const stateDir = stateDirFrom(values.state); - const { baseUrl, token } = loadCoolifyEnv(join(stateDir, ".coolify.env")); - const client = new CoolifyClient(baseUrl, token); + // Bindings first, but only when --env was given: an environment's + // `instance:` binding is what selects the Coolify to ask. Without --env + // there is no binding to read (and deliberately so — see below), so the + // flag or the default file decides. + const binding = values.env + ? loadBindings(join(stateDir, "environments.yaml")).environments[ + values.env + ] + : undefined; + if (values.env && !binding) { + console.error(`environment ${values.env} not in environments.yaml`); + return 2; + } + const { client } = openCoolify(stateDir, values.instance, binding); // Read-only, and the one command that deliberately does NOT require a // team binding: it is how you discover the values to write into // environments.yaml in the first place. Asserting here would be circular. @@ -530,13 +598,7 @@ async function main(): Promise { // "will apply refuse?" — ask the question without touching anything. const actual = await client.currentTeam(); console.log(`token's team: ${formatTeam(actual)}`); - if (!values.env) return 0; - const binding = loadBindings(join(stateDir, "environments.yaml")) - .environments[values.env]; - if (!binding) { - console.error(`environment ${values.env} not in environments.yaml`); - return 2; - } + if (!values.env || !binding) return 0; await assertTeam(client, binding.team, values.env); console.log(`matches the team ${values.env} expects ✓`); return 0; diff --git a/src/config.ts b/src/config.ts index d4f13cd..5cd5215 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,11 +1,28 @@ -import { readFileSync } from "node:fs"; +import { existsSync, readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; -// Coolify's base URL + API token, read from /.coolify.env — the one -// file in the state repo that is never committed (it is a live credential). -export function loadCoolifyEnv(path: string): { +// Which Coolify cast is about to talk to. The single most consequential input +// to any command — so it is a value that gets resolved, named, and printed, +// not an implicit property of whatever `.coolify.env` happens to contain right +// now. +export type CoolifyInstance = { + // "default" for /.coolify.env, else the --instance name. + name: string; baseUrl: string; token: string; -} { + // Declared by the instance, not inferred from the token: an instance + // configured for inspection must not be writable even if its token would + // permit the writes. See assertWritable. + readOnly: boolean; + // The file it came from, so every message can name it. + file: string; +}; + +export const DEFAULT_INSTANCE = "default"; +const DEFAULT_FILE = ".coolify.env"; +const INSTANCE_DIR = ".coolify"; + +function parseEnvFile(path: string): Record { const vars: Record = {}; for (const raw of readFileSync(path, "utf8").split("\n")) { const line = raw.trim(); @@ -14,6 +31,17 @@ export function loadCoolifyEnv(path: string): { if (eq === -1) continue; vars[line.slice(0, eq)] = line.slice(eq + 1).replace(/^"|"$/g, ""); } + return vars; +} + +// Coolify's base URL + API token, read from /.coolify.env — the one +// file in the state repo that is never committed (it is a live credential). +export function loadCoolifyEnv(path: string): { + baseUrl: string; + token: string; + readOnly: boolean; +} { + const vars = parseEnvFile(path); const baseUrl = vars.COOLIFY_BASE_URL; const token = vars.COOLIFY_ACCESS_TOKEN; if (!baseUrl || !token) { @@ -21,5 +49,82 @@ export function loadCoolifyEnv(path: string): { `${path}: COOLIFY_BASE_URL and COOLIFY_ACCESS_TOKEN are required`, ); } - return { baseUrl, token }; + return { baseUrl, token, readOnly: vars.COOLIFY_READ_ONLY === "true" }; +} + +// The named instances configured in a state dir: /.coolify/.env. +export function knownInstances(stateDir: string): string[] { + const dir = join(stateDir, INSTANCE_DIR); + if (!existsSync(dir)) return []; + return readdirSync(dir) + .filter((f) => f.endsWith(".env")) + .map((f) => f.slice(0, -".env".length)) + .sort(); +} + +export function instanceFile(stateDir: string, name?: string): string { + return name === undefined + ? join(stateDir, DEFAULT_FILE) + : join(stateDir, INSTANCE_DIR, `${name}.env`); +} + +// Refuse, don't guess — the same position `diff` takes on an absent target +// (#12). An unknown instance name is not a reason to fall back to the default +// one: "the instance I asked for isn't there, so I used a different one" is +// how a diff meant for a legacy box gets run against production. +export function loadInstance(stateDir: string, name?: string): CoolifyInstance { + const file = instanceFile(stateDir, name); + if (name !== undefined && !existsSync(file)) { + const known = knownInstances(stateDir); + throw new Error( + [ + `no Coolify instance named "${name}"`, + "", + ` looked for: ${file}`, + ` configured: ${known.join(", ") || "(none)"}`, + "", + "A named instance is an env file holding COOLIFY_BASE_URL +", + "COOLIFY_ACCESS_TOKEN (and optionally COOLIFY_READ_ONLY=true). Create the", + "file above, or pass one of the names that exist. With no --instance, cast", + `reads ${join(stateDir, DEFAULT_FILE)}.`, + ].join("\n"), + ); + } + const { baseUrl, token, readOnly } = loadCoolifyEnv(file); + return { + name: name ?? DEFAULT_INSTANCE, + baseUrl, + token, + readOnly, + file, + }; +} + +// A read-only instance is one the operator declared for inspection. The token +// it holds may well be able to write — that is exactly the point: this turns +// "I pointed the wrong token at the wrong box" from a live incident into an +// exit code, before the first mutating call rather than after it. +export function assertWritable(instance: CoolifyInstance, verb: string): void { + if (!instance.readOnly) return; + throw new Error( + [ + `refusing to ${verb}: Coolify instance "${instance.name}" is read-only`, + "", + ` declared by: COOLIFY_READ_ONLY=true in ${instance.file}`, + ` base url: ${instance.baseUrl}`, + "", + `\`${verb}\` writes. An instance configured for inspection cannot be written`, + "to, even if its token would permit it. Run `cast diff` or `cast team`", + "against this instance, or pass an --instance that is not read-only.", + ].join("\n"), + ); +} + +// What cast prints before it touches a Coolify, on every command that reaches +// one. The instance is the input most likely to be wrong and least likely to +// be noticed — so it gets said out loud, next to the team assert. +export function formatInstance(instance: CoolifyInstance): string { + return `instance ${instance.name} → ${instance.baseUrl}${ + instance.readOnly ? " (read-only)" : "" + }`; } diff --git a/test/cli.test.ts b/test/cli.test.ts index 1e8fb76..d15b20a 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -1,22 +1,107 @@ -import { execFileSync } from "node:child_process"; -import { describe, expect, it } from "vitest"; +import { spawn } from "node:child_process"; +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; -function runCli(args: string[]): { code: number; output: string } { - try { - const output = execFileSync("node", ["dist/cli.js", ...args], { - encoding: "utf8", - stdio: "pipe", +// Spawned ASYNCHRONOUSLY, and that is load-bearing: the stub Coolify below +// runs in THIS process, so a blocking execFileSync would hold the event loop +// and the stub could never answer the CLI it just launched — the two would +// deadlock until the test timed out. +function runCli(args: string[]): Promise<{ code: number; output: string }> { + return new Promise((resolve) => { + const child = spawn("node", ["dist/cli.js", ...args], { + stdio: ["ignore", "pipe", "pipe"], }); - return { code: 0, output }; - } catch (e) { - const err = e as { status: number; stderr: string; stdout: string }; - return { code: err.status, output: `${err.stdout}${err.stderr}` }; + let output = ""; + child.stdout.on("data", (d) => { + output += String(d); + }); + child.stderr.on("data", (d) => { + output += String(d); + }); + child.on("close", (code) => resolve({ code: code ?? 0, output })); + }); +} + +// A Coolify that answers the two calls these paths make, and RECORDS what it +// was asked. The recording is the point: "which instance did cast actually +// talk to" is the question #14 exists to make answerable, so the tests below +// answer it from the wire rather than from cast's own console output. +type Stub = { url: string; hits: string[]; close: () => Promise }; +const stubs: Stub[] = []; + +async function stubCoolify(): Promise { + const hits: string[] = []; + const server = createServer((req, res) => { + hits.push(req.url ?? ""); + const body = + req.url === "/api/v1/teams/current" + ? JSON.stringify({ id: 0, name: "Root Team" }) + : "[]"; + res.writeHead(200, { "content-type": "application/json" }); + res.end(body); + }); + await new Promise((r) => { + server.listen(0, "127.0.0.1", r); + }); + const stub: Stub = { + url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + hits, + close: () => + new Promise((r) => { + server.close(() => r()); + }), + }; + stubs.push(stub); + return stub; +} + +afterEach(async () => { + await Promise.all(stubs.splice(0).map((s) => s.close())); +}); + +const coolifyEnv = (url: string, readOnly = false) => + `COOLIFY_BASE_URL="${url}"\nCOOLIFY_ACCESS_TOKEN="t"\n${ + readOnly ? "COOLIFY_READ_ONLY=true\n" : "" + }`; + +// A state dir: the default instance, plus any named ones, plus bindings. +function stateWith(opts: { + default?: string; + named?: Record; + boundInstance?: string; +}): string { + const dir = mkdtempSync(join(tmpdir(), "cast-cli-")); + if (opts.default) writeFileSync(join(dir, ".coolify.env"), opts.default); + if (opts.named) { + mkdirSync(join(dir, ".coolify")); + for (const [name, body] of Object.entries(opts.named)) { + writeFileSync(join(dir, ".coolify", `${name}.env`), body); + } } + writeFileSync( + join(dir, "environments.yaml"), + [ + "environments:", + " prod:", + " server: prod-box", + " team: { id: 0, name: Root Team }", + ...(opts.boundInstance ? [` instance: ${opts.boundInstance}`] : []), + "github_apps:", + " incubator: hdb-coolify", + "smoke_target: core", + "", + ].join("\n"), + ); + return dir; } describe("infra cli", () => { - it("refuses apply --path with --env prod, exit non-zero", () => { - const r = runCli([ + it("refuses apply --path with --env prod, exit non-zero", async () => { + const r = await runCli([ "apply", "acme/widget", "--env", @@ -27,8 +112,8 @@ describe("infra cli", () => { expect(r.code).not.toBe(0); expect(r.output).toMatch(/--path.*prod/); }); - it("prints usage on unknown command", () => { - const r = runCli(["frobnicate"]); + it("prints usage on unknown command", async () => { + const r = await runCli(["frobnicate"]); expect(r.code).not.toBe(0); expect(r.output).toMatch(/usage: cast (apply|diff)/i); }); @@ -36,8 +121,8 @@ describe("infra cli", () => { // server into the token's team (permanently: a server belongs to exactly one // team), and `smoke` writes env vars onto a live app. Neither may run without // an environment to assert the token's team against. - it("refuses server add without --env, exit non-zero", () => { - const r = runCli([ + it("refuses server add without --env, exit non-zero", async () => { + const r = await runCli([ "server", "add", "prod-box", @@ -49,9 +134,125 @@ describe("infra cli", () => { expect(r.code).not.toBe(0); expect(r.output).toMatch(/--env/); }); - it("refuses smoke without --env, exit non-zero", () => { - const r = runCli(["smoke"]); + it("refuses smoke without --env, exit non-zero", async () => { + const r = await runCli(["smoke"]); expect(r.code).not.toBe(0); expect(r.output).toMatch(/--env/); }); }); + +describe("--instance (multiple Coolify instances)", () => { + // The acceptance criterion from #14, end to end: a command against a named + // instance reaches THAT Coolify, with no edit to .coolify.env — which still + // sits there, untouched, pointing somewhere else entirely. + it("talks to the named instance, leaving .coolify.env untouched", async () => { + const [main, legacy] = [await stubCoolify(), await stubCoolify()]; + const dir = stateWith({ + default: coolifyEnv(main.url), + named: { legacy: coolifyEnv(legacy.url) }, + }); + const r = await runCli(["team", "--state", dir, "--instance", "legacy"]); + expect(r.code).toBe(0); + expect(r.output).toContain(`instance legacy → ${legacy.url}`); + // The wire is the witness, not the log line. + expect(legacy.hits).toContain("/api/v1/teams/current"); + expect(main.hits).toEqual([]); + }); + + it("uses .coolify.env when no instance is named — unchanged behavior", async () => { + const [main, legacy] = [await stubCoolify(), await stubCoolify()]; + const dir = stateWith({ + default: coolifyEnv(main.url), + named: { legacy: coolifyEnv(legacy.url) }, + }); + const r = await runCli(["team", "--state", dir]); + expect(r.code).toBe(0); + expect(r.output).toContain(`instance default → ${main.url}`); + expect(main.hits).toContain("/api/v1/teams/current"); + expect(legacy.hits).toEqual([]); + }); + + // environments.yaml binds the instance, so --env prod selects the right + // control plane with no flag and no file edit at all. + it("honors an environment's instance binding with no flag", async () => { + const [main, prodCp] = [await stubCoolify(), await stubCoolify()]; + const dir = stateWith({ + default: coolifyEnv(main.url), + named: { "prod-cp": coolifyEnv(prodCp.url) }, + boundInstance: "prod-cp", + }); + const r = await runCli(["team", "--state", dir, "--env", "prod"]); + expect(r.code).toBe(0); + expect(r.output).toContain(`instance prod-cp → ${prodCp.url}`); + expect(prodCp.hits).toContain("/api/v1/teams/current"); + expect(main.hits).toEqual([]); + }); + + it("lets an explicit --instance beat the environment's binding", async () => { + const [prodCp, legacy] = [await stubCoolify(), await stubCoolify()]; + const dir = stateWith({ + named: { + "prod-cp": coolifyEnv(prodCp.url), + legacy: coolifyEnv(legacy.url), + }, + boundInstance: "prod-cp", + }); + const r = await runCli([ + "team", + "--state", + dir, + "--env", + "prod", + "--instance", + "legacy", + ]); + expect(r.code).toBe(0); + expect(legacy.hits).toContain("/api/v1/teams/current"); + expect(prodCp.hits).toEqual([]); + }); + + it("refuses an unknown instance, names the known ones, and touches nothing", async () => { + const main = await stubCoolify(); + const dir = stateWith({ + default: coolifyEnv(main.url), + named: { legacy: coolifyEnv(main.url) }, + }); + const r = await runCli(["team", "--state", dir, "--instance", "typo"]); + expect(r.code).not.toBe(0); + expect(r.output).toMatch(/no Coolify instance named "typo"/); + expect(r.output).toMatch(/configured:\s+legacy/); + // Refuse, don't guess: it must not have fallen back to the default. + expect(main.hits).toEqual([]); + }); + + // The nice-to-have that turns "I pointed the wrong token at the wrong box" + // into an exit code: the refusal lands BEFORE the first call, so a read-only + // instance never even gets asked. + it("refuses a writing verb against a read-only instance, before any call", async () => { + const legacy = await stubCoolify(); + const dir = stateWith({ + named: { legacy: coolifyEnv(legacy.url, true) }, + }); + const r = await runCli([ + "smoke", + "--state", + dir, + "--env", + "prod", + "--instance", + "legacy", + ]); + expect(r.code).not.toBe(0); + expect(r.output).toMatch(/refusing to smoke.*read-only/s); + expect(legacy.hits).toEqual([]); + }); + + it("still allows a read-only instance to be read", async () => { + const legacy = await stubCoolify(); + const dir = stateWith({ named: { legacy: coolifyEnv(legacy.url, true) } }); + const r = await runCli(["team", "--state", dir, "--instance", "legacy"]); + expect(r.code).toBe(0); + expect(r.output).toMatch(/read-only/); + expect(legacy.hits).toContain("/api/v1/teams/current"); + }); +}); diff --git a/test/config.test.ts b/test/config.test.ts new file mode 100644 index 0000000..20ee672 --- /dev/null +++ b/test/config.test.ts @@ -0,0 +1,143 @@ +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + assertWritable, + formatInstance, + knownInstances, + loadInstance, +} from "../src/config.js"; + +// A state dir with a default .coolify.env and any number of named instances. +function stateDir( + named: Record = {}, + defaultEnv?: string, +): string { + const dir = mkdtempSync(join(tmpdir(), "cast-state-")); + if (defaultEnv !== undefined) { + writeFileSync(join(dir, ".coolify.env"), defaultEnv); + } + if (Object.keys(named).length > 0) { + mkdirSync(join(dir, ".coolify")); + for (const [name, body] of Object.entries(named)) { + writeFileSync(join(dir, ".coolify", `${name}.env`), body); + } + } + return dir; +} + +const OK = + 'COOLIFY_BASE_URL="https://cp.example.com"\nCOOLIFY_ACCESS_TOKEN="t"\n'; + +describe("loadInstance", () => { + // The whole point of #14 is that adding this must change nothing for anyone + // who does not use it. + it("reads .coolify.env when no instance is named — unchanged behavior", () => { + const dir = stateDir({}, OK); + const inst = loadInstance(dir); + expect(inst).toMatchObject({ + name: "default", + baseUrl: "https://cp.example.com", + token: "t", + readOnly: false, + file: join(dir, ".coolify.env"), + }); + }); + + it("reads .coolify/.env for a named instance", () => { + const dir = stateDir( + { + legacy: + 'COOLIFY_BASE_URL="https://old.example.com"\nCOOLIFY_ACCESS_TOKEN="lt"\n', + }, + OK, + ); + expect(loadInstance(dir, "legacy")).toMatchObject({ + name: "legacy", + baseUrl: "https://old.example.com", + token: "lt", + }); + }); + + // Refuse, don't guess (#12's position on an absent target, applied to the + // connection target). Falling back to the default instance here is how a + // diff meant for a legacy box gets run against production. + it("refuses an unknown instance and names the ones that exist", () => { + const dir = stateDir({ "prod-cp": OK, "staging-cp": OK }, OK); + expect(() => loadInstance(dir, "legacy")).toThrow( + /no Coolify instance named "legacy"/, + ); + expect(() => loadInstance(dir, "legacy")).toThrow(/prod-cp, staging-cp/); + }); + + it("says so plainly when no named instances exist at all", () => { + const dir = stateDir({}, OK); + expect(() => loadInstance(dir, "legacy")).toThrow(/\(none\)/); + }); + + it("never silently falls back to the default instance", () => { + const dir = stateDir({}, OK); + expect(() => loadInstance(dir, "legacy")).toThrow(); + }); + + it("reads COOLIFY_READ_ONLY off an instance", () => { + const dir = stateDir({ legacy: `${OK}COOLIFY_READ_ONLY=true\n` }); + expect(loadInstance(dir, "legacy").readOnly).toBe(true); + }); + + it("still requires base url and token", () => { + const dir = stateDir({ broken: 'COOLIFY_BASE_URL="https://x"\n' }); + expect(() => loadInstance(dir, "broken")).toThrow( + /COOLIFY_BASE_URL and COOLIFY_ACCESS_TOKEN are required/, + ); + }); +}); + +describe("knownInstances", () => { + it("lists named instances, sorted, and is empty when there are none", () => { + expect(knownInstances(stateDir({ b: OK, a: OK }, OK))).toEqual(["a", "b"]); + expect(knownInstances(stateDir({}, OK))).toEqual([]); + }); +}); + +describe("assertWritable", () => { + const inst = (readOnly: boolean) => ({ + name: "legacy", + baseUrl: "https://old.example.com", + token: "t", + readOnly, + file: "/s/.coolify/legacy.env", + }); + + // "I pointed the wrong token at the wrong box" becomes an exit code rather + // than a live incident — and it holds even when the TOKEN would permit the + // write. That is the point: the declaration is the guard, not the scope. + it("refuses a write against a read-only instance, naming the declaration", () => { + expect(() => assertWritable(inst(true), "apply")).toThrow( + /refusing to apply.*read-only/s, + ); + expect(() => assertWritable(inst(true), "apply")).toThrow( + /COOLIFY_READ_ONLY=true in \/s\/\.coolify\/legacy\.env/, + ); + }); + + it("allows writes against a normal instance", () => { + expect(() => assertWritable(inst(false), "apply")).not.toThrow(); + }); +}); + +describe("formatInstance", () => { + it("names the instance and its base url, and flags read-only", () => { + const base = { + name: "legacy", + baseUrl: "https://old.example.com", + token: "t", + file: "f", + }; + expect(formatInstance({ ...base, readOnly: false })).toBe( + "instance legacy → https://old.example.com", + ); + expect(formatInstance({ ...base, readOnly: true })).toMatch(/read-only/); + }); +}); From 3eee70fa773ebc80f6c10a18091792efb067d740 Mon Sep 17 00:00:00 2001 From: claude-hdb Date: Mon, 13 Jul 2026 16:51:43 +0000 Subject: [PATCH 3/3] =?UTF-8?q?feat:=20cast=20capture=20=E2=80=94=20adopt?= =?UTF-8?q?=20a=20hand-built=20Coolify=20into=20the=20age=20secret=20store?= =?UTF-8?q?=20(#15)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cast was scoped to the steady state: manifest → Coolify, forever. It had no adoption path — no way to bootstrap the age store from an instance built by hand, before any manifest existed. The operator did it by hand: curl the envs, assemble 17 name=value pairs into /dev/shm/prod.env, age -r, shred. Every input to that pipeline is something cast already has, so a human was shuffling cast's own inputs through a terminal, with the leak (scrollback, history, a tmp file that never got shredded) and the silent miss both live. cast capture / --env [--generated N] [--override N] [--force] The required set comes from the MANIFEST, not the box: the ${...} refs in that environment's env templates, read by the same parser apply uses to demand them. resolveTemplate and templateRefs now share one grammar — a drift between them would mean capture collects a different set than apply later requires, which is exactly the "a name silently missed" failure this verb exists to remove. The mapping is deliberately NOT mechanical. A DATABASE_URL read off the source points at the SOURCE box's Postgres: confidently wrong, entirely plausible, and the target's real URL does not exist until Coolify creates the resource. So the manifest declares `generated_secrets:` and those names are written as the literal `pending-coolify-generated`. staging's ADMIN_EMAIL must be the operator, not the source's — staging and prod share a Mailgun domain, so a staging box carrying the real address can mail real users; that is --override. A "capture everything" verb would be wrong in ~4 of 17 entries, silently — worse than being wrong in all of them. So every name is forced into a disposition, and two of the four stop the run: a name required by a template but absent from the source REFUSES (an empty substitutes to nothing and the app boots misconfigured), as does one name carrying different values on two resources. generated_secrets is a manifest property rather than a flag the operator must remember, because the manifest is what knows DATABASE_URL comes from a database it declares. An entry no template refers to is a hard error: a guard standing over nothing reads like a guard, and the likeliest cause is a typo whose real name is then captured from the source instead of placeheld. Secret hygiene, all covered by tests asserting on real values: - the plan prints names and provenance, NEVER values - an --override's value comes from $CAST_CAPTURE_, never argv (`ps`) - plaintext is piped to age on stdin — never a temp file, stdout, or history - an existing store is not overwritten without --force: it may hold the only copy of values the source no longer has (apply's never-delete, applied here) capture inherits diff's absent-target refusal (D-237) — against a project that isn't there it would report every secret as missing, an alarming report about the wrong box — plus the team assert and the --path/--env prod ban. The last gate is a typed confirmation of the environment's name; there is no --yes. The end-to-end test decrypts the store cast wrote and asserts on its contents, so "exactly the names the manifest requires, no more and no fewer" is checked against real ciphertext rather than against cast's own console output. Co-Authored-By: Claude Opus 4.8 --- README.md | 121 +++++++++++++- docs/semantics.md | 126 +++++++++++++++ src/bindings.ts | 6 + src/capture.ts | 189 ++++++++++++++++++++++ src/cli.ts | 275 ++++++++++++++++++++++++++++---- src/envtemplate.ts | 56 +++++-- src/manifest.ts | 13 ++ src/resolve.ts | 78 ++++++++- src/secrets.ts | 21 +++ test/capture-cli.test.ts | 334 +++++++++++++++++++++++++++++++++++++++ test/capture.test.ts | 309 ++++++++++++++++++++++++++++++++++++ 11 files changed, 1478 insertions(+), 50 deletions(-) create mode 100644 src/capture.ts create mode 100644 test/capture-cli.test.ts create mode 100644 test/capture.test.ts diff --git a/README.md b/README.md index 22990ba..f5243f3 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ environments.yaml # bindings: the team each env's token must belon # GitHub App name, smoke target, guards secrets/..env.age # age-encrypted values for the ${…} placeholders .coolify.env # COOLIFY_BASE_URL + COOLIFY_ACCESS_TOKEN (never commit) +.coolify/.env # …the same, for a NAMED instance (see below) ``` Pass it with `--state `, or set `CAST_STATE`. Defaults to the cwd. @@ -59,8 +60,9 @@ Pass it with `--state `, or set `CAST_STATE`. Defaults to the cwd. ## Commands ```sh -cast apply / --env [--path ] [--hostname-overlay ] -cast diff / --env [--full] +cast apply / --env [--path ] [--hostname-overlay ] +cast diff / --env [--full] +cast capture / --env [--generated ] [--override ] cast server add --ip --key --env [--user root] [--port 22] cast smoke --env cast team [--env ] @@ -73,6 +75,9 @@ cast team [--env ] default branch). - **`diff`** — reports drift, manifest → Coolify. Structural by default; `--full` also compares env vars. Exits non-zero when dirty, so CI can gate on it. +- **`capture`** — the adoption path: reads a hand-built instance's live env and + writes the environment's age store from it. See *Adopting a hand-built + instance* below. - **`server add`** — uploads a server's private key and registers it with Coolify. - **`smoke`** — contract test against `smoke_target`: proves Coolify's bulk env endpoint still *upserts* rather than replacing. Run it after every Coolify @@ -89,6 +94,118 @@ them first asserts the token's team (below). `--hostname-overlay` swaps domains for a pre-flight run against temporary hostnames; re-applying **without** it is the cutover. +## Cloning: cast authenticates, and never prompts + +`apply`, `diff` and `capture` clone the product repo (unless `--path` points at a +local checkout — refused for prod, which always reads the default branch). For a +private repo that needs credentials, and cast resolves them itself: + +1. **`gh`**, borrowed as a credential helper for that one invocation — it does + not touch your global git config. +2. **`GITHUB_TOKEN` / `GH_TOKEN`** from the environment (the CI path). +3. Whatever git's own credential helper does, if you have one. + +Being logged into `gh` is enough. You do **not** need `gh auth setup-git` — +that separate act is what wires git's helper, and not running it is exactly how +you end up at git's interactive username/password prompt, which GitHub no longer +accepts. cast sets `GIT_TERMINAL_PROMPT=0` on every path, so it can never hang +there or hide a credentials failure behind an error about *the repository*. With +no credentials at all it says so, and names the fix. + +The token is never put in the clone URL or in `http.extraheader` — both leak it +into `ps`, and the latter persists it into the clone's git config. + +## Many Coolifys + +`--instance ` reads `/.coolify/.env` instead of +`/.coolify.env`. Every verb that reaches Coolify takes it. + +```sh +cast diff heavy-duty/incubator --env prod --full --instance legacy +``` + +An environment can bind one, so `--env` selects the right control plane with no +flag at all: + +```yaml +environments: + prod: + server: prod-box + team: { id: 1, name: heavy-duty } + instance: prod-cp # → /.coolify/prod-cp.env +``` + +An explicit `--instance` still wins, so a one-off read against a legacy box needs +no edit to that file either. **With no flag and no binding, nothing changes** — +`.coolify.env` is read exactly as before. + +Two properties, both deliberate: + +- **An unknown `--instance` refuses**, and names the instances that do exist. + Falling back to the default is how a diff meant for a legacy box gets run + against production. +- **An instance may declare `COOLIFY_READ_ONLY=true`**, and then `apply`, + `smoke` and `server add` refuse it — *before their first call*, and even + though the token itself would permit the writes. That turns "I pointed the + wrong token at the wrong box" from a live incident into an exit code. + +Every command that reaches a Coolify now says which one, next to the team +assert. It is the most consequential input to any run, and the least visible. + +## Adopting a hand-built instance + +cast is otherwise scoped to the steady state: manifest → Coolify, forever. +`capture` is the one-way-in — it bootstraps an environment's age store from an +instance that was built by hand, before any manifest existed. + +```sh +CAST_CAPTURE_ADMIN_EMAIL=me@example.com \ + cast capture heavy-duty/incubator --env prod --instance legacy \ + --override ADMIN_EMAIL +``` + +It reads the required secret **names** from the manifest's own env templates (the +`${…}` refs — the manifest already declares exactly this set), reads the live +values off the instance, and classifies every name: + +| | | +| --- | --- | +| **captured** | found live, value taken | +| **generated** | the manifest's `generated_secrets` declares it provider-made → written as the literal `pending-coolify-generated`, never the live value | +| **overridden** | supplied by you, for a value that must *not* be carried over | +| **missing** | required by a template, absent live → **refuses** | + +Then it prints a plan of **names and provenance — never values** — and waits for +you to type the environment's name. + +The mapping is not mechanical, and that is the whole design. A `DATABASE_URL` +copied off the source box points at the *source box's* Postgres: confidently +wrong, entirely plausible, and the target's real URL does not exist until Coolify +creates the resource. So the manifest declares those names, and cast placeholds +them: + +```yaml +environments: + prod: + generated_secrets: [DATABASE_URL_PROD, REDIS_URL_PROD, UMAMI_DATABASE_URL] +``` + +It is a manifest property rather than a flag you have to remember, because the +manifest is what knows `DATABASE_URL` comes from a database it declares. (A +`generated_secrets` entry no template refers to is a schema error — a guard +standing over nothing is worse than no guard, because it reads like one. +`--generated ` covers a manifest that hasn't declared them yet.) + +An **`--override`**'s value is read from `$CAST_CAPTURE_`, never from the +command line: argv is visible in `ps` to every process on the box. It exists for +values that must not survive the copy — staging and prod sharing a Mailgun +domain means a staging box carrying the real `ADMIN_EMAIL` can mail real users. + +The store is encrypted to the environment's `age_recipient` (add it to +`environments.yaml` — it's the public half, safe to commit). Plaintext goes to +`age` on stdin: it is never a temp file, never on stdout, never in your shell +history. An existing store is not overwritten without `--force`. + **[docs/semantics.md](docs/semantics.md)** is the contract behind those commands: what `apply` guarantees (never deletes, never recreates a database, fails loudly rather than recreating on un-updatable drift), the `dockercompose` diff --git a/docs/semantics.md b/docs/semantics.md index bff133b..735f3e2 100644 --- a/docs/semantics.md +++ b/docs/semantics.md @@ -158,6 +158,132 @@ softened by an implementation detail): refuses `--path` combined with `--env prod`: prod always reads the default branch, so a feature-branch checkout can never reach it. +## Instance selection + +**The Coolify a command talks to is an explicit, named value** — not a property +of whatever `/.coolify.env` happens to contain at the moment. Resolution +order, highest first: + +1. `--instance ` → `/.coolify/.env` +2. the environment's `instance:` binding in `environments.yaml` +3. `/.coolify.env` (the default; unchanged when neither of the above is + used) + +Two refusals, both fail-closed: + +- **An unknown `--instance` aborts**, naming the instances that do exist. It + does *not* fall back to the default — that fallback is how a `--full` diff + meant for a legacy box gets run against production. +- **`COOLIFY_READ_ONLY=true` in an instance file makes it read-only**, and + `apply` / `smoke` / `server add` refuse it *before their first call*. The + guard is the **declaration**, not the token's scope: an instance configured + for inspection must not be writable even when the token it holds would permit + the writes. `diff`, `team` and `capture` still work against it — they read. + +Every command that reaches a live Coolify prints which one, next to the team +assert. + +## Adoption (`capture`) + +`capture` is the only verb that writes *into* the state directory rather than +into Coolify, and the only one that reads a hand-built instance as a **source** +rather than as a target. It exists because cast is otherwise scoped to the +steady state and has no bootstrap path for a box that predates its manifest. + +**The required set comes from the manifest, not from the box.** The names are +the `${…}` refs in that environment's env templates, read by the same parser +`apply` uses to demand them (`parseTemplate`, shared by `resolveTemplate` and +`templateRefs` — deliberately one grammar, because a drift between the two +would mean `capture` collects a different set than `apply` will later require, +which is the "a name silently missed" failure it exists to remove). So the store +it writes contains **exactly** the names the manifest requires: a live var +nobody asked for is not the store's business, and a template literal +(`NODE_ENV=production`) is not a secret. + +**The mapping is not mechanical, and must not be.** Some entries encode +migration decisions rather than facts about the source box: + +- A `DATABASE_URL` / `REDIS_URL` read off the source points at the **source + box's** Postgres/Redis. Copying it is confidently wrong in a way that looks + entirely plausible, and the target's real URL does not exist until Coolify + creates the resource. These are declared `generated_secrets:` in the manifest + environment and written as the literal `pending-coolify-generated`. +- staging's `ADMIN_EMAIL` must be the operator, not the source's value: staging + and prod share a Mailgun domain, so a staging box carrying the real address + can mail real users. That is `--override`. + +A "capture everything" verb would therefore be silently wrong in a handful of +entries out of seventeen — worse than being wrong in all of them. So every +required name is **forced into a disposition**, and two of the four stop the +run: + +| disposition | source | outcome | +| --- | --- | --- | +| captured | found live | value taken | +| generated | manifest `generated_secrets` (or `--generated`) | `pending-coolify-generated` | +| overridden | `$CAST_CAPTURE_` | operator's value | +| **missing** | required by a template, absent live | **refuses** | +| **conflict** | one name, different live values on two resources | **refuses** | + +`generated_secrets` is a **manifest** property, not a flag: the manifest is what +knows `DATABASE_URL` comes from a database it declares. An entry naming +something no template refs is a hard error — dead config here is not untidy but +dangerous, because it reads like a guard standing over a name while standing +over nothing, and the likeliest cause is a typo whose real name is then +*captured* from the source box instead of placeheld. + +**Secret hygiene**, all enforced by tests against real values: + +- The plan prints **names and provenance, never values**. (The one value-shaped + thing it prints is the `pending-coolify-generated` literal, which carries no + information about the source.) +- An `--override`'s value is read from `$CAST_CAPTURE_`, **never from + argv** — a command-line value is visible in `ps` to every process on the box. +- Plaintext is piped to `age` on **stdin**: never a temp file, never stdout, + never shell history. The hand-run recipe this replaces wrote + `/dev/shm/prod.env` and relied on remembering to `shred -u` it. +- An existing store is **not overwritten** without `--force`: it may hold the + only copy of values the source box no longer has. Same disposition as apply's + never-delete. + +**`capture` takes `diff`'s position on an absent target** (see `LiveLookup`), +and refuses one: against a project or environment that isn't there it would read +back zero live values and report every required secret as *missing* — an +alarming, meaningless report about the wrong box. It also inherits the team +assert (a wrong-team token reads back `null` for everything, producing the same +lie) and the `--path`-with-`--env prod` refusal (a feature-branch manifest must +not decide which names land in the prod store). + +The final gate is a **typed confirmation** — the environment's own name, after +the plan. There is no `--yes`: a store written without someone reading the +provenance column is the outcome the verb exists to prevent. A closed stdin +aborts rather than hanging. + +## Cloning a private manifest + +`resolveCheckout` resolves git credentials **inside cast**, in a fixed order — +`gh` borrowed as a per-invocation credential helper, then +`GITHUB_TOKEN`/`GH_TOKEN`, then the ambient helper — rather than leaving it to +whatever the workstation's git config happens to do. + +It matters because `gh auth login` **does not** wire git's credential helper +(that is `gh auth setup-git`, a separate act most people never run), so a +perfectly logged-in operator still fell through to git's interactive +username/password prompt — which GitHub no longer accepts — and got an error +about *the repository* rather than about the missing credentials. There is no +routing around it for prod: `--path` is refused there, so the clone is the only +path and its auth is mandatory. + +`GIT_TERMINAL_PROMPT=0` is set on every path, so cast can never hang on or fall +into that prompt. The token is never placed in the clone URL or in +`http.extraheader` — both leak it into `ps`, and the latter persists it into the +clone's `.git/config`; the helper reads it from the environment at run time, so +what lands in argv is the literal text `$CAST_GIT_TOKEN`. Note that the empty +`credential.helper=` reset clears **URL-scoped** helpers +(`credential.https://github.com.helper`, which is what `gh auth setup-git` +writes) as well as generic ones, so cast's chosen credential is genuinely the +one used — verified against a live private clone. + **Known limitations, not defects:** - **Backup schedules are create-time only.** A manifest database's `backup` diff --git a/src/bindings.ts b/src/bindings.ts index c66d301..b8ce0af 100644 --- a/src/bindings.ts +++ b/src/bindings.ts @@ -47,6 +47,12 @@ const BindingsSchema = z // An explicit --instance still wins, so a one-off read against a // legacy box needs no change to this file either. instance: z.string().optional(), + // The age recipient (public key) this environment's secret store is + // encrypted TO. Only `capture` needs it — decryption resolves an + // identity per keyFileFor, and the state repo deliberately holds + // ciphertext but never the identity that opens it. This is the + // public half, so it is safe to commit here next to the bindings. + age_recipient: z.string().optional(), s3_destination: z.string().optional(), // Var-name patterns this environment refuses outright (see // assertEnvVarPolicy). Operator-owned guard: prod typically bans diff --git a/src/capture.ts b/src/capture.ts new file mode 100644 index 0000000..958dcfc --- /dev/null +++ b/src/capture.ts @@ -0,0 +1,189 @@ +import type { RequiredSecret } from "./resolve.js"; + +// What the manifest writes for a provider-generated name. Not the source box's +// value — that points at the SOURCE box's Postgres/Redis — and not an empty +// string, which would boot the app misconfigured. A literal that is obviously +// a placeholder, and that Coolify replaces when it creates the resource. +export const GENERATED_PLACEHOLDER = "pending-coolify-generated"; + +// Live env vars, per resource: resource name -> (env key -> value). +export type LiveEnvs = Record>; + +export type Provenance = "captured" | "generated" | "overridden"; + +export type Site = { resource: string; key: string }; + +export type Disposition = { + ref: string; + provenance: Provenance; + // Never rendered. Kept here so the caller can encrypt it, and nowhere else. + value: string; + sites: Site[]; +}; + +export type Classification = { + plan: Disposition[]; + // Required by a template, absent from the source, and not dispositioned + // otherwise. Refuses the run: writing an empty value substitutes to nothing + // and the app boots misconfigured — the exact failure capture exists to + // remove, and one that looks entirely plausible from the outside. + missing: Array<{ ref: string; sites: Site[] }>; + // The same ref carrying DIFFERENT live values on two resources. cast cannot + // pick, and picking wrong is silent, so it refuses. + conflicts: Array<{ ref: string; values: Site[] }>; +}; + +function groupByRef(required: RequiredSecret[]): Map { + const byRef = new Map(); + for (const { ref, resource, key } of required) { + const sites = byRef.get(ref) ?? []; + sites.push({ resource, key }); + byRef.set(ref, sites); + } + return byRef; +} + +// Force disposition, never guess. Every name the manifest requires lands in +// exactly one of four buckets, and two of them stop the run. +// +// The mapping is deliberately NOT a mechanical dump of the source box: some +// entries encode migration decisions rather than facts about the source. A +// "capture everything" verb would be wrong in a handful of entries out of +// seventeen, silently — which is worse than being wrong in all of them. +export function classify( + required: RequiredSecret[], + generated: string[], + live: LiveEnvs, + overrides: Record, +): Classification { + const generatedSet = new Set(generated); + const plan: Disposition[] = []; + const missing: Classification["missing"] = []; + const conflicts: Classification["conflicts"] = []; + + for (const [ref, sites] of groupByRef(required)) { + // The operator's word beats both the manifest and the source box: this is + // the escape hatch for a value that must NOT be carried over (staging's + // ADMIN_EMAIL, where the source's value is a real founder and staging + // shares a Mailgun domain with prod). + if (ref in overrides) { + plan.push({ + ref, + provenance: "overridden", + value: overrides[ref], + sites, + }); + continue; + } + if (generatedSet.has(ref)) { + plan.push({ + ref, + provenance: "generated", + value: GENERATED_PLACEHOLDER, + sites, + }); + continue; + } + // Captured: read the live value off whichever resources declare it. + const found = sites + .map((s) => ({ site: s, value: live[s.resource]?.[s.key] })) + .filter((f): f is { site: Site; value: string } => f.value !== undefined); + if (found.length === 0) { + missing.push({ ref, sites }); + continue; + } + const distinct = new Set(found.map((f) => f.value)); + if (distinct.size > 1) { + conflicts.push({ ref, values: found.map((f) => f.site) }); + continue; + } + plan.push({ + ref, + provenance: "captured", + value: found[0].value, + sites, + }); + } + return { plan, missing, conflicts }; +} + +const site = (s: Site) => `${s.resource}.${s.key}`; + +// Names and provenance. NEVER values. +// +// The one thing printed that looks like a value is GENERATED_PLACEHOLDER, +// which is a literal constant in this file and carries no information about +// the source box. Everything else is a name the manifest already declares in +// plaintext, in a committed file. +export function renderCapturePlan( + c: Classification, + ctx: { + orgRepo: string; + env: string; + instance: string; + store: string; + recipient: string; + }, +): string { + const lines = [ + `capture plan — ${ctx.orgRepo} ${ctx.env}`, + "", + ` source: instance ${ctx.instance} (live values read from it)`, + ` store: ${ctx.store}`, + ` recipient: ${ctx.recipient}`, + "", + ]; + const width = Math.max( + 0, + ...[...c.plan, ...c.missing, ...c.conflicts].map((d) => d.ref.length), + ); + for (const d of c.plan) { + const where = d.sites.map(site).join(", "); + const note = + d.provenance === "generated" + ? ` → ${GENERATED_PLACEHOLDER}` + : d.provenance === "overridden" + ? ` (from CAST_CAPTURE_${d.ref})` + : ""; + lines.push( + ` ${d.ref.padEnd(width)} ${d.provenance.padEnd(10)} ${where}${note}`, + ); + } + for (const m of c.missing) { + lines.push( + ` ${m.ref.padEnd(width)} MISSING required by ${m.sites.map(site).join(", ")}, absent from the source`, + ); + } + for (const c2 of c.conflicts) { + lines.push( + ` ${c2.ref.padEnd(width)} CONFLICT differs between ${c2.values.map(site).join(" and ")}`, + ); + } + const counts = (["captured", "generated", "overridden"] as const) + .map((p) => [p, c.plan.filter((d) => d.provenance === p).length] as const) + .filter(([, n]) => n > 0) + .map(([p, n]) => `${n} ${p}`) + .join(", "); + lines.push( + "", + `${c.plan.length} name(s) to write${counts ? `: ${counts}` : ""}`, + ); + if (c.missing.length > 0) { + lines.push( + "", + `refusing to write the store: ${c.missing.length} name(s) the manifest requires are not`, + "present on the source. An empty value substitutes to nothing and the app boots", + "misconfigured — plausibly, and silently. Supply each one with --override ", + "(its value is read from CAST_CAPTURE_, never from argv), or fix the source.", + ); + } + if (c.conflicts.length > 0) { + lines.push( + "", + `refusing to write the store: ${c.conflicts.length} name(s) carry different values on`, + "different resources of the source. The store holds one value per name, and cast", + "will not pick for you. Reconcile them on the source, or pin one with --override.", + ); + } + return lines.join("\n"); +} diff --git a/src/cli.ts b/src/cli.ts index b261da9..56039ac 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,10 +1,12 @@ #!/usr/bin/env node -import { readFileSync } from "node:fs"; +import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; +import { createInterface } from "node:readline/promises"; import { parseArgs } from "node:util"; import { parse as parseYaml } from "yaml"; import { type Executor, applyHostnameOverlay, applyPlan } from "./apply.js"; import { githubAppNameFor, loadBindings } from "./bindings.js"; +import { type LiveEnvs, classify, renderCapturePlan } from "./capture.js"; import { type CoolifyInstance, assertWritable, @@ -19,14 +21,24 @@ import { renderDiff, } from "./diff.js"; import { assertEnvVarPolicy } from "./envtemplate.js"; -import { desiredFromManifest, resolveCheckout } from "./resolve.js"; -import { decryptSecrets, keyFileFor, secretsFileFor } from "./secrets.js"; +import { + desiredFromManifest, + requiredSecrets, + resolveCheckout, +} from "./resolve.js"; +import { + decryptSecrets, + encryptSecrets, + keyFileFor, + secretsFileFor, +} from "./secrets.js"; import { serverAdd } from "./server.js"; import { smoke } from "./smoke.js"; import { assertTeam, formatTeam } from "./team.js"; -const USAGE = `usage: cast apply / --env [--path ] [--project ] [--hostname-overlay ] - cast diff / --env [--full] [--project ] +const USAGE = `usage: cast apply / --env [--path ] [--project ] [--hostname-overlay ] + cast diff / --env [--full] [--project ] + cast capture / --env [--path ] [--project ] [--generated ] [--override ] [--force] cast server add --ip --key --env [--user root] [--port 22] cast smoke --env cast team [--env ] @@ -50,7 +62,16 @@ const USAGE = `usage: cast apply / --env [--path ] [--proj repo (the default). A project built by hand in the UI is called whatever someone typed; \`diff\` refuses rather than reporting an absent project as an empty one, and this is how you point it at - the real name.`; + the real name. + +capture (adopt a hand-built instance into the age secret store): + --generated force NAME to the \`pending-coolify-generated\` placeholder, + for a manifest that has not declared generated_secrets yet. + Repeatable. + --override supply NAME yourself instead of copying the source's value. + The VALUE is read from \$CAST_CAPTURE_, never from the + command line — argv is visible in \`ps\`. Repeatable. + --force overwrite an existing store (refused by default).`; // cast is stateless: every instance-scoped input is read from the state // directory it is pointed at, never from a location the tool itself knows. @@ -290,21 +311,26 @@ export async function fetchLive( // exists next to it. export function renderAbsentTarget( lookup: Extract, - ctx: { orgRepo: string; overridden: boolean }, + ctx: { orgRepo: string; overridden: boolean; verb?: string }, ): string { + // `capture` takes the same position as `diff`, and for the same reason: it + // is only ever a claim about something that already exists. Against an + // absent target it would read back zero live values and call every required + // secret "missing" — an alarming-but-meaningless report about the wrong box. + const verb = ctx.verb ?? "diff"; const origin = ctx.overridden ? "--project" : `derived from the repo slug ${ctx.orgRepo}`; const head = lookup.missing === "project" ? [ - `refusing to diff: no project named "${lookup.project}" exists in this team`, + `refusing to ${verb}: no project named "${lookup.project}" exists in this team`, "", ` looked for: project "${lookup.project}" (${origin})`, ` exists here: ${lookup.available.join(", ") || "(no projects at all)"}`, ] : [ - `refusing to diff: project "${lookup.project}" has no environment "${lookup.environment}"`, + `refusing to ${verb}: project "${lookup.project}" has no environment "${lookup.environment}"`, "", ` looked for: environment "${lookup.environment}" in project "${lookup.project}"`, " note: cast names environments after --env, so a project built by", @@ -316,7 +342,7 @@ export function renderAbsentTarget( "", "An absent target reads back exactly like an empty one, so continuing would diff", 'it as "nothing exists — create everything": a clean-looking report that verified', - "nothing. `apply` may create a target; `diff` may only ever describe one that is", + `nothing. \`apply\` may create a target; \`${verb}\` may only ever describe one that is`, "already there.", "", lookup.missing === "project" @@ -325,6 +351,79 @@ export function renderAbsentTarget( ].join("\n"); } +// A live resource's env vars, by key. `real_value` is the decrypted one and +// needs a token with read:sensitive; `value` is what a lesser token sees. +// +// A 404 (a resource we just listed no longer having an envs endpoint — not +// expected in practice, but consistent with treating "gone" as "no env vars") +// collapses to {}; anything else (401, 5xx, network) must surface. Swallowing +// it would make a live resource's env look EMPTY, which turns every one of its +// vars into a spurious create in a diff, and into a spurious "missing" in a +// capture. +async function fetchEnv( + client: CoolifyClient, + l: Live, +): Promise> { + const base = l.kind === "database" ? "databases" : `${l.kind}s`; + const envs = (await client.get(`/${base}/${l.uuid}/envs`).catch((err) => { + if (err instanceof HttpError && err.status === 404) return []; + throw err; + })) as Array<{ key: string; real_value?: string; value: string }>; + return Object.fromEntries(envs.map((e) => [e.key, e.real_value ?? e.value])); +} + +// The value for an --override, read from the ENVIRONMENT rather than argv. +// +// A secret passed as a command-line argument is visible in `ps` to every +// process on the box, and lands in shell history — the same class of leak the +// clone-auth fix (#13) exists to avoid. So --override names the secret and the +// environment carries it. +function readOverrides(names: string[]): Record { + const out: Record = {}; + for (const name of names) { + const varName = `CAST_CAPTURE_${name}`; + const value = process.env[varName]; + if (value === undefined) { + throw new Error( + [ + `--override ${name}: no value supplied.`, + "", + `cast reads an override's value from ${varName}, never from the command`, + "line — an argv value is visible in `ps` to every process on this box.", + "", + ` ${varName}=… cast capture …`, + ].join("\n"), + ); + } + out[name] = value; + } + return out; +} + +// Typed confirmation, and deliberately NOT a --yes flag. +// +// This verb writes an environment's secret store, once, off a box nobody is +// going to rebuild. The entire reason it exists is that the hand-run version +// was easy to get subtly wrong — so the last gate is a human who has read the +// provenance column typing the environment's own name. Nothing shorter counts: +// not "y", not a flag. Automating it means deliberately echoing the +// environment name into cast, which is an explicit act rather than an absent +// one. +// +// EOF (a closed or empty stdin) resolves to `null` and aborts. Without that +// race, a `< /dev/null` run would hang forever on a question nobody can answer. +async function confirmCapture(envName: string): Promise { + const rl = createInterface({ input: process.stdin, output: process.stdout }); + const answer = await new Promise((resolve) => { + rl.question( + `\ntype the environment name to write this store (${envName}): `, + ).then(resolve, () => resolve(null)); + rl.once("close", () => resolve(null)); + }); + rl.close(); + return answer?.trim() === envName; +} + async function main(): Promise { const [command, ...rest] = process.argv.slice(2); if (command === "-h" || command === "--help" || command === "help") { @@ -419,28 +518,7 @@ async function main(): Promise { const live = lookup.found ? lookup.live : []; if (mode === "full") { for (const l of live) { - const envs = (await client - .get( - `/${l.kind === "database" ? "databases" : `${l.kind}s`}/${l.uuid}/envs`, - ) - .catch((err) => { - // Same policy as fetchLive's environment fetch: a 404 (a - // resource we just listed no longer having an envs endpoint — - // not expected in practice, but consistent with treating - // "gone" as "no env vars") collapses to []; anything else - // (401, 5xx, network) must surface. Swallowing it here would - // make a live resource's env look empty and turn every one of - // its vars into a spurious create in the diff. - if (err instanceof HttpError && err.status === 404) return []; - throw err; - })) as Array<{ - key: string; - real_value?: string; - value: string; - }>; - l.env = Object.fromEntries( - envs.map((e) => [e.key, e.real_value ?? e.value]), - ); + l.env = await fetchEnv(client, l); } } const report = computeDiff(desired, live, mode); @@ -466,6 +544,139 @@ async function main(): Promise { ); return 0; } + if (command === "capture") { + const { values, positionals } = parseArgs({ + args: rest, + allowPositionals: true, + options: { + env: { type: "string" }, + state: { type: "string" }, + path: { type: "string" }, + project: { type: "string" }, + instance: { type: "string" }, + generated: { type: "string", multiple: true }, + override: { type: "string", multiple: true }, + force: { type: "boolean", default: false }, + }, + }); + const orgRepo = positionals[0]; + const envName = values.env; + if (!orgRepo || !envName) { + console.error(USAGE); + return 2; + } + const stateDir = stateDirFrom(values.state); + const repoShort = orgRepo.split("/")[1]; + const projectName = values.project ?? repoShort; + const store = secretsFileFor(stateDir, repoShort, envName); + // Never overwrite a store by accident. `apply` never deletes; the verb + // that WRITES the store gets the same disposition, because the thing it + // would destroy is the only copy of values that may not exist anywhere + // else any more. + if (existsSync(store) && !values.force) { + console.error( + [ + `refusing to capture: ${store} already exists`, + "", + "That store may hold the only copy of values the source box no longer has.", + "Pass --force to overwrite it deliberately, or move it aside first.", + ].join("\n"), + ); + return 2; + } + const bindings = loadBindings(join(stateDir, "environments.yaml")); + const binding = bindings.environments[envName]; + if (!binding) { + console.error(`environment ${envName} not in environments.yaml`); + return 2; + } + const recipient = binding.age_recipient; + if (!recipient) { + console.error( + [ + `environment ${envName} has no age_recipient in environments.yaml`, + "", + "capture encrypts the store TO that recipient (the public half of the", + "environment's age key — safe to commit next to the bindings). Add it:", + "", + " environments:", + ` ${envName}:`, + " age_recipient: age1…", + ].join("\n"), + ); + return 2; + } + // Same rule as apply (resolveCheckout enforces it): prod always reads the + // default branch. A feature-branch manifest must not be able to decide + // which names land in the prod store. + const checkout = resolveCheckout(orgRepo, { + env: envName, + path: values.path, + }); + const { required, generated } = requiredSecrets(checkout, envName); + const overrides = readOverrides(values.override ?? []); + const { client } = openCoolify(stateDir, values.instance, binding); + // capture READS Coolify and writes only to the local store, so it is + // allowed against a read-only instance — inspecting a legacy box is + // precisely what such an instance is for. It still takes the team assert: + // a wrong-team token reads back nothing, and "nothing" here would render + // as "every secret is missing" against a box that is fine. + const team = await assertTeam(client, binding.team, envName); + console.log(`team ${formatTeam(team)} ✓`); + const lookup = await fetchLive(client, projectName, envName); + if (!lookup.found) { + console.error( + renderAbsentTarget(lookup, { + orgRepo, + overridden: values.project !== undefined, + verb: "capture", + }), + ); + return 2; + } + const liveEnvs: LiveEnvs = {}; + for (const l of lookup.live) { + // Databases hold no manifest-templated env of their own — their URL is + // what the APPS reference, and that name is generated, not captured. + if (l.kind === "database") continue; + liveEnvs[l.name] = await fetchEnv(client, l); + } + const classification = classify( + required, + [...generated, ...(values.generated ?? [])], + liveEnvs, + overrides, + ); + console.log( + renderCapturePlan(classification, { + orgRepo, + env: envName, + instance: values.instance ?? binding.instance ?? "default", + store, + recipient, + }), + ); + // Refuse, don't write a wrong store. Both of these are stop conditions, + // and the plan above has already named every offending entry. + if ( + classification.missing.length > 0 || + classification.conflicts.length > 0 + ) + return 2; + if (!(await confirmCapture(envName))) { + console.error("aborted — nothing written"); + return 2; + } + encryptSecrets( + recipient, + store, + Object.fromEntries(classification.plan.map((d) => [d.ref, d.value])), + ); + console.log( + `wrote ${store} — ${classification.plan.length} name(s), encrypted to ${recipient}`, + ); + return 0; + } if (command === "server" && rest[0] === "add") { const { values, positionals } = parseArgs({ args: rest.slice(1), diff --git a/src/envtemplate.ts b/src/envtemplate.ts index 53cd5f6..0c2f5b5 100644 --- a/src/envtemplate.ts +++ b/src/envtemplate.ts @@ -2,11 +2,18 @@ export type ResolvedEnv = { vars: Record; }; -export function resolveTemplate( - text: string, - secrets: Record, -): ResolvedEnv { - const vars: ResolvedEnv["vars"] = {}; +// A template line, parsed but not resolved: `ref` is set when the whole RHS is +// a single ${NAME} placeholder. +export type TemplateVar = { key: string; rhs: string; ref?: string }; + +// ONE grammar, shared by both readers of a template — resolveTemplate (which +// needs the values) and templateRefs (which needs only the names). Keeping +// them on separate parsers would let the two drift, and a drift here is not +// cosmetic: `capture` would collect a different set of names than `apply` will +// later demand, which is precisely the "a name silently missed" failure the +// capture verb exists to remove. +function parseTemplate(text: string): TemplateVar[] { + const vars: TemplateVar[] = []; const lines = text.split("\n"); for (let i = 0; i < lines.length; i++) { const line = lines[i].trim(); @@ -18,21 +25,42 @@ export function resolveTemplate( ); const [, key, rhs] = m; const placeholder = rhs.match(/^\$\{([A-Z][A-Z0-9_]*)\}$/); - if (placeholder) { - const value = secrets[placeholder[1]]; - if (value === undefined) { - throw new Error( - `secret ${placeholder[1]} (for ${key}) missing from the age store`, - ); - } - vars[key] = { value, secret: true }; - } else { + vars.push({ key, rhs, ...(placeholder ? { ref: placeholder[1] } : {}) }); + } + return vars; +} + +export function resolveTemplate( + text: string, + secrets: Record, +): ResolvedEnv { + const vars: ResolvedEnv["vars"] = {}; + for (const { key, rhs, ref } of parseTemplate(text)) { + if (ref === undefined) { vars[key] = { value: rhs, secret: false }; + continue; } + const value = secrets[ref]; + if (value === undefined) { + throw new Error(`secret ${ref} (for ${key}) missing from the age store`); + } + vars[key] = { value, secret: true }; } return { vars }; } +// The ${NAME} refs a template declares: the secret names the manifest requires, +// paired with the env var each one lands on. `capture` reads these to learn +// what to go and fetch — at capture time there is no store to resolve against +// yet, which is the whole point of the verb. +export function templateRefs( + text: string, +): Array<{ key: string; ref: string }> { + return parseTemplate(text).flatMap(({ key, ref }) => + ref === undefined ? [] : [{ key, ref }], + ); +} + // An environment may forbid variables by name pattern, declared as // `environments..forbidden_var_patterns` in the state repo. The rule is // PRESENCE, not value: a forbidden var set to "false" still refuses the apply, diff --git a/src/manifest.ts b/src/manifest.ts index e22768f..1e4df6b 100644 --- a/src/manifest.ts +++ b/src/manifest.ts @@ -82,6 +82,19 @@ const EnvironmentSpecSchema = z applications: z.record(AppSpecSchema), databases: z.record(DatabaseSpecSchema).optional(), services: z.record(ServiceSpecSchema).optional(), + // Secret names whose values the PROVIDER generates — a Coolify-created + // Postgres/Redis URL, a service's own generated credentials. `capture` + // writes these as the literal `pending-coolify-generated` and never copies + // the source box's live value: that value points at the SOURCE box's + // database, so carrying it over would be confidently wrong in a way that + // looks entirely plausible, and the target's real URL does not exist until + // Coolify creates the resource. + // + // It is a manifest property rather than a flag the operator has to + // remember, because the manifest is what knows DATABASE_URL comes from a + // database it declares. Optional: a manifest that names none simply has no + // generated secrets, and `capture` will say so in its plan. + generated_secrets: z.array(z.string()).optional(), }) .strict(); diff --git a/src/resolve.ts b/src/resolve.ts index c76012a..799261a 100644 --- a/src/resolve.ts +++ b/src/resolve.ts @@ -3,7 +3,11 @@ import { existsSync, mkdtempSync, readFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { Desired } from "./diff.js"; -import { type ResolvedEnv, resolveTemplate } from "./envtemplate.js"; +import { + type ResolvedEnv, + resolveTemplate, + templateRefs, +} from "./envtemplate.js"; import { loadManifest } from "./manifest.js"; // How cast authenticated (or failed to authenticate) a clone. @@ -132,8 +136,11 @@ export function resolveCheckout( opts: { env: string; path?: string }, ): string { if (opts.path && opts.env === "prod") { + // Holds for every verb that reads a manifest (apply, and now capture): a + // feature-branch checkout must not be able to decide what prod runs, nor + // which secret names land in prod's store. throw new Error( - "apply refuses --path with --env prod: prod always reads the default branch", + "refuses --path with --env prod: prod always reads the default branch", ); } if (opts.path) return opts.path; @@ -171,6 +178,73 @@ export function resolveCheckout( return dir; } +// One secret the manifest requires: the ${REF} a template names (the key it +// gets in the age store), the resource that needs it, and the env var it lands +// on there. That last pair is what `capture` reads the live value from — the +// store is keyed by REF, but the live box knows it as `resource.key`. +export type RequiredSecret = { ref: string; resource: string; key: string }; + +// Exactly the set of secret names an environment's manifest demands — the same +// set `apply` will later insist on, read from the same templates by the same +// parser. `capture` uses this to know what to go and fetch; nothing else has to +// be told, and nothing can be silently missed. +// +// Deliberately does NOT take a secrets map: at capture time the store does not +// exist yet. That is the whole point of the verb. +export function requiredSecrets( + checkoutDir: string, + envName: string, +): { required: RequiredSecret[]; generated: string[] } { + const manifest = loadManifest(join(checkoutDir, ".infra", "manifest.yaml")); + const envSpec = manifest.environments[envName]; + if (!envSpec) { + throw new Error( + `environment ${envName} not in manifest (has: ${Object.keys(manifest.environments).join(", ") || "none"})`, + ); + } + const required: RequiredSecret[] = []; + const collect = (resource: string, template?: string) => { + if (!template) return; + const file = join(checkoutDir, ".infra", "env", template); + if (!existsSync(file)) + throw new Error( + `env template missing: ${file} (referenced by ${resource})`, + ); + for (const { key, ref } of templateRefs(readFileSync(file, "utf8"))) { + required.push({ ref, resource, key }); + } + }; + for (const [name, app] of Object.entries(envSpec.applications)) { + collect(name, app.env_template); + } + for (const [name, svc] of Object.entries(envSpec.services ?? {})) { + collect(name, svc.env_template); + } + const generated = envSpec.generated_secrets ?? []; + // A generated_secrets entry naming something no template refs is dead + // config — and dead config in THIS list is not merely untidy, it is + // dangerous: it reads like a guard standing over a name while standing over + // nothing. The likeliest cause is a typo, and the consequence of the typo is + // that the real name gets CAPTURED from the source box instead of placeheld. + const refs = new Set(required.map((r) => r.ref)); + const dead = generated.filter((g) => !refs.has(g)); + if (dead.length > 0) { + throw new Error( + [ + `manifest environment ${envName}: generated_secrets names ${dead.join(", ")}, which no env template refers to`, + "", + ` declared: ${generated.join(", ")}`, + ` templates: ${[...refs].sort().join(", ") || "(no ${...} refs at all)"}`, + "", + "A generated name that matches nothing guards nothing — and if this is a", + "typo, the name it was meant to guard is being captured from the source", + "box instead of placeheld. Fix the spelling, or drop the entry.", + ].join("\n"), + ); + } + return { required, generated }; +} + export function desiredFromManifest( checkoutDir: string, envName: string, diff --git a/src/secrets.ts b/src/secrets.ts index 63ca0f5..bb4ab9e 100644 --- a/src/secrets.ts +++ b/src/secrets.ts @@ -22,6 +22,27 @@ export function decryptSecrets( return secrets; } +// Write an environment's secret store, encrypted to its recipient. +// +// The plaintext goes to age on STDIN and the ciphertext straight to `file`: it +// is never a temp file, never reaches the terminal, and never lands in shell +// history. The hand-run recipe this replaces assembled /dev/shm/prod.env and +// relied on remembering to `shred -u` it afterwards — a step that is invisible +// when it is skipped. +export function encryptSecrets( + recipient: string, + file: string, + vars: Record, +): void { + const plaintext = `${Object.entries(vars) + .map(([k, v]) => `${k}=${v}`) + .join("\n")}\n`; + execFileSync("age", ["-r", recipient, "-o", file], { + input: plaintext, + stdio: ["pipe", "pipe", "pipe"], + }); +} + // The age identity for an environment, resolved without cast knowing anything // about your environment names: // diff --git a/test/capture-cli.test.ts b/test/capture-cli.test.ts new file mode 100644 index 0000000..a45c42e --- /dev/null +++ b/test/capture-cli.test.ts @@ -0,0 +1,334 @@ +import { execFileSync, spawn } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + writeFileSync, +} from "node:fs"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeAll, describe, expect, it } from "vitest"; +import { decryptSecrets } from "../src/secrets.js"; + +// End-to-end: the real CLI, a real age identity, a stub Coolify holding real +// live values. The point is the store that comes out the other side — it is +// decrypted and asserted on, so "exactly the names the manifest requires, no +// more and no fewer" is checked against the actual ciphertext rather than +// against cast's own console output. + +const SECRETS = { + // Points at the SOURCE box. Must NOT be carried over. + DATABASE_URL: "postgres://user:pw@SOURCE-BOX-postgres:5432/app", + MAILGUN_API_KEY: "key-REAL-MAILGUN-SECRET", + OPENROUTER_API_KEY: "sk-or-REAL-OPENROUTER-SECRET", + // A real founder. Must NOT be carried over to staging. + ADMIN_EMAIL: "founder@real-company.com", + // Live on the box, but the manifest never asks for it. + UNRELATED_LIVE_VAR: "nobody-asked-for-this", +}; + +let keyFile: string; +let recipient: string; + +beforeAll(() => { + const dir = mkdtempSync(join(tmpdir(), "cast-age-")); + keyFile = join(dir, "age-staging.key"); + execFileSync("age-keygen", ["-o", keyFile], { stdio: "pipe" }); + const pub = execFileSync("age-keygen", ["-y", keyFile], { encoding: "utf8" }); + recipient = pub.trim(); +}); + +type Stub = { url: string; close: () => Promise }; +const stubs: Stub[] = []; + +// A Coolify with one project, one environment, one application carrying the +// live env above. +async function stubCoolify(): Promise { + const server = createServer((req, res) => { + const path = (req.url ?? "").replace("/api/v1", ""); + const json = (body: unknown) => { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify(body)); + }; + if (path === "/teams/current") return json({ id: 0, name: "Root Team" }); + if (path === "/projects") return json([{ uuid: "p1", name: "incubator" }]); + if (path === "/projects/p1/staging") + return json({ applications: [{ name: "core", uuid: "a1" }] }); + if (path === "/applications/a1/envs") + return json( + Object.entries(SECRETS).map(([key, real_value]) => ({ + key, + real_value, + value: "REDACTED", + })), + ); + res.writeHead(404); + res.end("{}"); + }); + await new Promise((r) => { + server.listen(0, "127.0.0.1", r); + }); + const stub: Stub = { + url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + close: () => + new Promise((r) => { + server.close(() => r()); + }), + }; + stubs.push(stub); + return stub; +} + +afterEach(async () => { + await Promise.all(stubs.splice(0).map((s) => s.close())); +}); + +const MANIFEST = `project: incubator +environments: + staging: + generated_secrets: [DATABASE_URL_STAGING] + applications: + core: + source: { repo: heavy-duty/incubator, branch: main } + build: { pack: nixpacks, base_directory: / } + domains: ["http://core.example.com"] + env_template: core.staging.env.template +`; + +// NODE_ENV is a literal, not a secret — it must not reach the store. +const TEMPLATE = `NODE_ENV=production +DATABASE_URL=\${DATABASE_URL_STAGING} +MAILGUN_API_KEY=\${MAILGUN_API_KEY} +OPENROUTER_API_KEY=\${OPENROUTER_API_KEY} +ADMIN_EMAIL=\${ADMIN_EMAIL} +`; + +function fixture(url: string, opts: { template?: string } = {}) { + const checkout = mkdtempSync(join(tmpdir(), "cast-co-")); + mkdirSync(join(checkout, ".infra", "env"), { recursive: true }); + writeFileSync(join(checkout, ".infra", "manifest.yaml"), MANIFEST); + writeFileSync( + join(checkout, ".infra", "env", "core.staging.env.template"), + opts.template ?? TEMPLATE, + ); + + const state = mkdtempSync(join(tmpdir(), "cast-state-")); + mkdirSync(join(state, "secrets")); + writeFileSync( + join(state, ".coolify.env"), + `COOLIFY_BASE_URL="${url}"\nCOOLIFY_ACCESS_TOKEN="t"\n`, + ); + writeFileSync( + join(state, "environments.yaml"), + [ + "environments:", + " staging:", + " server: staging-box", + " team: { id: 0, name: Root Team }", + ` age_recipient: ${recipient}`, + "github_apps:", + " incubator: hdb-coolify", + "", + ].join("\n"), + ); + return { + checkout, + state, + store: join(state, "secrets", "incubator.staging.env.age"), + }; +} + +function runCapture( + args: string[], + opts: { stdin?: string; env?: Record } = {}, +): Promise<{ code: number; output: string }> { + return new Promise((resolve) => { + const child = spawn("node", ["dist/cli.js", "capture", ...args], { + stdio: ["pipe", "pipe", "pipe"], + env: { ...process.env, ...opts.env }, + }); + let output = ""; + child.stdout.on("data", (d) => { + output += String(d); + }); + child.stderr.on("data", (d) => { + output += String(d); + }); + child.stdin.end(opts.stdin ?? ""); + child.on("close", (code) => resolve({ code: code ?? 0, output })); + }); +} + +const base = (f: ReturnType) => [ + "heavy-duty/incubator", + "--env", + "staging", + "--state", + f.state, + "--path", + f.checkout, +]; + +describe("cast capture (end to end)", () => { + it("writes a store with exactly the manifest's names, and the right provenance", async () => { + const f = fixture((await stubCoolify()).url); + const r = await runCapture([...base(f), "--override", "ADMIN_EMAIL"], { + stdin: "staging\n", + env: { CAST_CAPTURE_ADMIN_EMAIL: "operator@example.com" }, + }); + expect(r.code).toBe(0); + expect(existsSync(f.store)).toBe(true); + + const store = decryptSecrets(f.store, keyFile); + // No more, no fewer: the four ${...} refs. NODE_ENV is a literal, and + // UNRELATED_LIVE_VAR is live but unasked-for — neither belongs here. + expect(Object.keys(store).sort()).toEqual([ + "ADMIN_EMAIL", + "DATABASE_URL_STAGING", + "MAILGUN_API_KEY", + "OPENROUTER_API_KEY", + ]); + // Captured verbatim. + expect(store.MAILGUN_API_KEY).toBe(SECRETS.MAILGUN_API_KEY); + expect(store.OPENROUTER_API_KEY).toBe(SECRETS.OPENROUTER_API_KEY); + // Generated: the placeholder, NEVER the source box's own database URL. + expect(store.DATABASE_URL_STAGING).toBe("pending-coolify-generated"); + expect(store.DATABASE_URL_STAGING).not.toContain("SOURCE-BOX"); + // Overridden: the operator's value, not the real founder's address. + expect(store.ADMIN_EMAIL).toBe("operator@example.com"); + expect(store.ADMIN_EMAIL).not.toBe(SECRETS.ADMIN_EMAIL); + }); + + // "No secret value is ever written to stdout" — checked against the real + // values the stub served, on the real console output of a real run. + it("never prints a secret value to the console", async () => { + const f = fixture((await stubCoolify()).url); + const r = await runCapture([...base(f), "--override", "ADMIN_EMAIL"], { + stdin: "staging\n", + env: { CAST_CAPTURE_ADMIN_EMAIL: "operator@example.com" }, + }); + expect(r.code).toBe(0); + for (const value of Object.values(SECRETS)) { + expect(r.output).not.toContain(value); + } + expect(r.output).not.toContain("operator@example.com"); + // It did print the NAMES, though — that is the plan. + expect(r.output).toContain("MAILGUN_API_KEY"); + expect(r.output).toContain("captured"); + expect(r.output).toContain("generated"); + expect(r.output).toContain("overridden"); + }); + + // The plaintext exists only in memory and on age's stdin. + it("leaves no plaintext behind — the store is real ciphertext", async () => { + const f = fixture((await stubCoolify()).url); + await runCapture([...base(f), "--override", "ADMIN_EMAIL"], { + stdin: "staging\n", + env: { CAST_CAPTURE_ADMIN_EMAIL: "operator@example.com" }, + }); + const raw = readFileSync(f.store, "utf8"); + expect(raw).toContain("age-encryption.org"); + for (const value of Object.values(SECRETS)) { + expect(raw).not.toContain(value); + } + }); + + // A name required by the template but absent from the source refuses the run + // — writing an empty would boot the app misconfigured, plausibly. + it("refuses when a required name is absent from the source", async () => { + const f = fixture((await stubCoolify()).url, { + template: `${TEMPLATE}TURNSTILE_SECRET=\${TURNSTILE_SECRET}\n`, + }); + const r = await runCapture([...base(f), "--override", "ADMIN_EMAIL"], { + stdin: "staging\n", + env: { CAST_CAPTURE_ADMIN_EMAIL: "operator@example.com" }, + }); + expect(r.code).not.toBe(0); + expect(r.output).toMatch(/TURNSTILE_SECRET\s+MISSING/); + expect(r.output).toMatch(/refusing to write the store/); + expect(existsSync(f.store)).toBe(false); + }); + + // The confirmation is the last gate, and it is not "y". + it("aborts, writing nothing, when the confirmation does not name the env", async () => { + const f = fixture((await stubCoolify()).url); + const r = await runCapture([...base(f), "--override", "ADMIN_EMAIL"], { + stdin: "y\n", + env: { CAST_CAPTURE_ADMIN_EMAIL: "operator@example.com" }, + }); + expect(r.code).not.toBe(0); + expect(r.output).toMatch(/aborted/); + expect(existsSync(f.store)).toBe(false); + }); + + it("aborts on a closed stdin rather than hanging", async () => { + const f = fixture((await stubCoolify()).url); + const r = await runCapture([...base(f), "--override", "ADMIN_EMAIL"], { + stdin: "", + env: { CAST_CAPTURE_ADMIN_EMAIL: "operator@example.com" }, + }); + expect(r.code).not.toBe(0); + expect(r.output).toMatch(/aborted/); + }); + + // An override's VALUE never comes from argv — argv is visible in `ps`. + it("refuses an --override whose CAST_CAPTURE_ is unset", async () => { + const f = fixture((await stubCoolify()).url); + const r = await runCapture([...base(f), "--override", "ADMIN_EMAIL"], { + stdin: "staging\n", + }); + expect(r.code).not.toBe(0); + expect(r.output).toMatch(/CAST_CAPTURE_ADMIN_EMAIL/); + expect(r.output).toMatch(/never from the command/); + }); + + // The store may hold the only copy of values the source box no longer has. + it("refuses to overwrite an existing store without --force", async () => { + const f = fixture((await stubCoolify()).url); + writeFileSync(f.store, "PRE-EXISTING"); + const r = await runCapture([...base(f), "--override", "ADMIN_EMAIL"], { + stdin: "staging\n", + env: { CAST_CAPTURE_ADMIN_EMAIL: "operator@example.com" }, + }); + expect(r.code).not.toBe(0); + expect(r.output).toMatch(/already exists/); + expect(r.output).toMatch(/--force/); + expect(readFileSync(f.store, "utf8")).toBe("PRE-EXISTING"); + }); + + it("refuses an environment with no age_recipient", async () => { + const f = fixture((await stubCoolify()).url); + writeFileSync( + join(f.state, "environments.yaml"), + [ + "environments:", + " staging:", + " server: staging-box", + " team: { id: 0, name: Root Team }", + "github_apps:", + " incubator: hdb-coolify", + "", + ].join("\n"), + ); + const r = await runCapture(base(f), { stdin: "staging\n" }); + expect(r.code).not.toBe(0); + expect(r.output).toMatch(/age_recipient/); + }); + + // Same position as diff: capture is only ever a claim about something that + // already exists. Against an absent target it would call every secret + // "missing" — an alarming report about the wrong box. + it("refuses an absent target rather than reporting every secret missing", async () => { + const f = fixture((await stubCoolify()).url); + const r = await runCapture([...base(f), "--project", "typo"], { + stdin: "staging\n", + }); + expect(r.code).not.toBe(0); + expect(r.output).toMatch(/refusing to capture/); + expect(r.output).toMatch(/no project named "typo"/); + expect(r.output).not.toMatch(/MISSING/); + }); +}); diff --git a/test/capture.test.ts b/test/capture.test.ts new file mode 100644 index 0000000..8118428 --- /dev/null +++ b/test/capture.test.ts @@ -0,0 +1,309 @@ +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + GENERATED_PLACEHOLDER, + classify, + renderCapturePlan, +} from "../src/capture.js"; +import { requiredSecrets } from "../src/resolve.js"; + +const CTX = { + orgRepo: "heavy-duty/incubator", + env: "prod", + instance: "legacy", + store: "/s/secrets/incubator.prod.env.age", + recipient: "age1abc", +}; + +// The live case, shrunk: an app whose env template needs a generated database +// URL, a carried-over API key, and an address that must NOT be carried over. +const REQUIRED = [ + { ref: "DATABASE_URL_PROD", resource: "core", key: "DATABASE_URL" }, + { ref: "MAILGUN_API_KEY", resource: "core", key: "MAILGUN_API_KEY" }, + { ref: "ADMIN_EMAIL", resource: "core", key: "ADMIN_EMAIL" }, +]; +const LIVE = { + core: { + DATABASE_URL: "postgres://SOURCE-BOX-INTERNAL/db", + MAILGUN_API_KEY: "key-abc123-REAL-SECRET", + ADMIN_EMAIL: "founder@real-company.com", + }, +}; + +describe("classify", () => { + it("captures a live value, and records where it came from", () => { + const c = classify([REQUIRED[1]], [], LIVE, {}); + expect(c.plan).toEqual([ + { + ref: "MAILGUN_API_KEY", + provenance: "captured", + value: "key-abc123-REAL-SECRET", + sites: [{ resource: "core", key: "MAILGUN_API_KEY" }], + }, + ]); + }); + + // The failure this verb exists to prevent: the source box's DATABASE_URL + // points at the SOURCE box's Postgres. Copying it over is confidently wrong + // in a way that looks entirely plausible. + it("placeholds a generated name, never copying the source's value", () => { + const c = classify([REQUIRED[0]], ["DATABASE_URL_PROD"], LIVE, {}); + expect(c.plan[0]).toMatchObject({ + ref: "DATABASE_URL_PROD", + provenance: "generated", + value: GENERATED_PLACEHOLDER, + }); + expect(c.plan[0].value).not.toContain("SOURCE-BOX"); + }); + + // staging and prod share a Mailgun domain, so a staging box carrying the + // real ADMIN_EMAIL can mail real users. + it("takes an override from the operator, over the source's value", () => { + const c = classify([REQUIRED[2]], [], LIVE, { + ADMIN_EMAIL: "operator@example.com", + }); + expect(c.plan[0]).toMatchObject({ + provenance: "overridden", + value: "operator@example.com", + }); + }); + + it("an override beats a generated declaration too", () => { + const c = classify([REQUIRED[0]], ["DATABASE_URL_PROD"], LIVE, { + DATABASE_URL_PROD: "postgres://explicit", + }); + expect(c.plan[0]).toMatchObject({ + provenance: "overridden", + value: "postgres://explicit", + }); + }); + + // Required by the template, absent from the source: refuse rather than write + // an empty. An empty substitutes to nothing and the app boots misconfigured. + it("refuses on a name required by the template but absent from the source", () => { + const c = classify( + [{ ref: "TURNSTILE_SECRET", resource: "core", key: "TURNSTILE_SECRET" }], + [], + LIVE, + {}, + ); + expect(c.plan).toEqual([]); + expect(c.missing).toEqual([ + { + ref: "TURNSTILE_SECRET", + sites: [{ resource: "core", key: "TURNSTILE_SECRET" }], + }, + ]); + }); + + it("a missing name can be rescued by an override", () => { + const c = classify( + [{ ref: "TURNSTILE_SECRET", resource: "core", key: "TURNSTILE_SECRET" }], + [], + LIVE, + { TURNSTILE_SECRET: "supplied" }, + ); + expect(c.missing).toEqual([]); + expect(c.plan[0].provenance).toBe("overridden"); + }); + + // One name, two resources, two different live values. The store holds one + // value per name; picking wrong would be silent. + it("refuses when one name carries different values on two resources", () => { + const c = classify( + [ + { ref: "SHARED", resource: "core", key: "SHARED" }, + { ref: "SHARED", resource: "worker", key: "SHARED" }, + ], + [], + { core: { SHARED: "a" }, worker: { SHARED: "b" } }, + {}, + ); + expect(c.plan).toEqual([]); + expect(c.conflicts).toEqual([ + { + ref: "SHARED", + values: [ + { resource: "core", key: "SHARED" }, + { resource: "worker", key: "SHARED" }, + ], + }, + ]); + }); + + it("is fine when one name carries the SAME value on two resources", () => { + const c = classify( + [ + { ref: "SHARED", resource: "core", key: "SHARED" }, + { ref: "SHARED", resource: "worker", key: "SHARED" }, + ], + [], + { core: { SHARED: "same" }, worker: { SHARED: "same" } }, + {}, + ); + expect(c.conflicts).toEqual([]); + expect(c.plan[0].value).toBe("same"); + }); + + // The acceptance criterion: exactly the names the manifest requires, no more + // and no fewer. A live var the manifest does not ask for is not the store's + // business. + it("writes exactly the required names — ignoring live vars nobody asked for", () => { + const c = classify( + REQUIRED, + ["DATABASE_URL_PROD"], + { + core: { ...LIVE.core, SOME_OTHER_LIVE_VAR: "not in the manifest" }, + }, + {}, + ); + expect(c.plan.map((d) => d.ref).sort()).toEqual([ + "ADMIN_EMAIL", + "DATABASE_URL_PROD", + "MAILGUN_API_KEY", + ]); + }); +}); + +describe("renderCapturePlan", () => { + // THE invariant. "No secret value is ever written to stdout" — so the plan + // is names and provenance, and the test asserts on the actual live values + // rather than on a pattern that could drift away from them. + it("never prints a secret value", () => { + const c = classify(REQUIRED, ["DATABASE_URL_PROD"], LIVE, { + ADMIN_EMAIL: "operator@example.com", + }); + const out = renderCapturePlan(c, CTX); + for (const secret of [ + "postgres://SOURCE-BOX-INTERNAL/db", + "key-abc123-REAL-SECRET", + "founder@real-company.com", + "operator@example.com", + ]) { + expect(out).not.toContain(secret); + } + }); + + it("shows every name with its provenance and where it lands", () => { + const c = classify(REQUIRED, ["DATABASE_URL_PROD"], LIVE, { + ADMIN_EMAIL: "operator@example.com", + }); + const out = renderCapturePlan(c, CTX); + expect(out).toMatch(/MAILGUN_API_KEY\s+captured\s+core\.MAILGUN_API_KEY/); + expect(out).toMatch(/DATABASE_URL_PROD\s+generated/); + expect(out).toContain(GENERATED_PLACEHOLDER); + expect(out).toMatch(/ADMIN_EMAIL\s+overridden/); + expect(out).toContain("CAST_CAPTURE_ADMIN_EMAIL"); + expect(out).toMatch(/3 name\(s\) to write/); + expect(out).toContain("/s/secrets/incubator.prod.env.age"); + expect(out).toContain("age1abc"); + }); + + it("names what is missing, and says why an empty would be worse", () => { + const c = classify( + [{ ref: "TURNSTILE_SECRET", resource: "core", key: "TURNSTILE_SECRET" }], + [], + LIVE, + {}, + ); + const out = renderCapturePlan(c, CTX); + expect(out).toMatch(/TURNSTILE_SECRET\s+MISSING/); + expect(out).toMatch(/refusing to write the store/); + expect(out).toMatch(/--override/); + }); + + it("names a conflict rather than picking a side", () => { + const c = classify( + [ + { ref: "SHARED", resource: "core", key: "SHARED" }, + { ref: "SHARED", resource: "worker", key: "SHARED" }, + ], + [], + { core: { SHARED: "a" }, worker: { SHARED: "b" } }, + {}, + ); + const out = renderCapturePlan(c, CTX); + expect(out).toMatch(/SHARED\s+CONFLICT/); + expect(out).toMatch(/core\.SHARED and worker\.SHARED/); + expect(out).not.toMatch(/\ba\b.*\bb\b/); + }); +}); + +// requiredSecrets is what makes "no more, no fewer" true: the set comes from +// the manifest's own templates, read by the same parser apply uses. +describe("requiredSecrets", () => { + function checkout(manifest: string, templates: Record) { + const dir = mkdtempSync(join(tmpdir(), "cast-cap-")); + mkdirSync(join(dir, ".infra", "env"), { recursive: true }); + writeFileSync(join(dir, ".infra", "manifest.yaml"), manifest); + for (const [name, body] of Object.entries(templates)) { + writeFileSync(join(dir, ".infra", "env", name), body); + } + return dir; + } + + const MANIFEST = `project: incubator +environments: + prod: + generated_secrets: [DATABASE_URL_PROD] + applications: + core: + source: { repo: heavy-duty/incubator, branch: main } + build: { pack: dockercompose, base_directory: /, compose_file: docker-compose.yaml } + service_domains: + api: ["https://api.example.com"] + env_template: core.prod.env.template + services: + umami: + type: umami + env_template: umami.prod.env.template +`; + + it("collects the ${...} refs from every app and service template", () => { + const dir = checkout(MANIFEST, { + "core.prod.env.template": + "NODE_ENV=production\nDATABASE_URL=${DATABASE_URL_PROD}\nMAILGUN_API_KEY=${MAILGUN_API_KEY}\n", + "umami.prod.env.template": "APP_SECRET=${UMAMI_APP_SECRET}\n", + }); + const { required, generated } = requiredSecrets(dir, "prod"); + expect(required).toEqual([ + { ref: "DATABASE_URL_PROD", resource: "core", key: "DATABASE_URL" }, + { ref: "MAILGUN_API_KEY", resource: "core", key: "MAILGUN_API_KEY" }, + { ref: "UMAMI_APP_SECRET", resource: "umami", key: "APP_SECRET" }, + ]); + expect(generated).toEqual(["DATABASE_URL_PROD"]); + }); + + // A non-placeholder line (NODE_ENV=production) is not a secret and must not + // land in the store — the store holds the ${...} refs, nothing else. + it("ignores literal template values — only ${...} refs are secrets", () => { + const dir = checkout(MANIFEST, { + "core.prod.env.template": + "NODE_ENV=production\nREPORTING_ENABLED=false\nDATABASE_URL=${DATABASE_URL_PROD}\n", + "umami.prod.env.template": "", + }); + const { required } = requiredSecrets(dir, "prod"); + expect(required.map((r) => r.ref)).toEqual(["DATABASE_URL_PROD"]); + }); + + // Dead config in THIS list is dangerous, not merely untidy: it reads like a + // guard standing over a name while standing over nothing, and the likeliest + // cause is a typo whose real name is then CAPTURED from the source box. + it("refuses a generated_secrets entry that no template refers to", () => { + const dir = checkout( + MANIFEST.replace( + "generated_secrets: [DATABASE_URL_PROD]", + "generated_secrets: [DATABASE_URL_TYPO]", + ), + { + "core.prod.env.template": "DATABASE_URL=${DATABASE_URL_PROD}\n", + "umami.prod.env.template": "", + }, + ); + expect(() => requiredSecrets(dir, "prod")).toThrow( + /generated_secrets names DATABASE_URL_TYPO/, + ); + }); +});