Merge pull request #24 from claude-hdb/feat/inventory-sweep

inventory sweeps the instance — and an empty environment shouts (#22)

README.md

@@ -76,12 +76,13 @@ cast team [--env <env>]
   default branch).
 - **`diff`** — reports drift, manifest → Coolify. Structural by default; `--full`
   also compares env vars. Exits non-zero when dirty, so CI can gate on it.
-- **`inventory`** — what is actually *on* a box, and how it lines up with the
+- **`inventory`** — what is actually *on* a box. **With no repo it sweeps the
-  manifest: resources and env var **keys** (never values), sorted into
+  instance** (every project, every environment, every resource — no manifest
-  on-both / manifest-only / box-only. Needs no store, no age key, and no
+  involved); with a repo it reconciles, showing resources and env var **keys**
-  recipient — it runs *before* adoption, which is the point of it. A document,
+  (never values) sorted into on-both / manifest-only / box-only. Needs no store,
-  read by a person; nothing here is consumed by `apply`. See *Adopting a
+  no age key, and no recipient — it runs *before* adoption, which is the point of
-  hand-built instance*.
+  it. A document, read by a person; nothing here is consumed by `apply`. See
+  *Adopting a hand-built instance*.
 - **`capture`** — the adoption path: reads a hand-built instance's live env and
   writes the environment's age store from it. See *Adopting a hand-built
   instance* below.
@@ -173,9 +174,40 @@ clicked *New Resource* that afternoon. `inventory` shows you both sides at once,
 so those differences arrive together, as a document — instead of one at a time,
 as refusals from a verb that is already halfway through a migration.
 
+**First, sweep it — you cannot aim at coordinates you do not have yet:**
+
+```sh
+cast inventory --env prod --instance legacy
+```
+
+```
+sweep — instance legacy (https://coolify.example.com)
+
+  Incubator
+    production     (empty)
+    staging        2 applications, 2 databases, 1 service
+      application  Incubator Stack v2
+      application  Incubator Landing
+      database     Incubator Database v2
+      …
+  La Familia Site
+    production     1 application
+      application  lafamilia-web
+```
+
+Note what that costs you to *not* have: Coolify auto-creates a `production`
+environment in every project, so the obvious guess is empty and the live system
+is somewhere else entirely — under a name someone typed, in a project you may
+not have known was there. An environment with **zero** resources is far more
+often the wrong coordinate than an empty one, and `inventory` says so rather than
+quietly reporting that the manifest has five things the box lacks.
+
+**Then reconcile**, against a target you now know exists:
+
 ```sh
 cast inventory heavy-duty/incubator --env prod --instance legacy \
-  --project Incubator --environment production
+  --project Incubator --environment staging \
+  --resource core="Incubator Stack v2"
 ```
 
 It never reads a value, needs no store and no key, and its output is **not**

src/cli.ts

@@ -28,7 +28,14 @@ import {
   renderDiff,
 } from "./diff.js";
 import { assertEnvVarPolicy } from "./envtemplate.js";
-import { type LiveResource, reconcile, renderInventory } from "./inventory.js";
+import {
+  type LiveResource,
+  type SweepEnvironment,
+  type SweepProject,
+  reconcile,
+  renderInventory,
+  renderSweep,
+} from "./inventory.js";
 import {
   desiredFromManifest,
   manifestResources,
@@ -48,7 +55,8 @@ import { assertTeam, formatTeam } from "./team.js";
 const USAGE = `usage: cast apply     <org>/<repo> --env <env> [--path <dir>] [--project <name>] [--environment <name>] [--hostname-overlay <file>]
        cast diff      <org>/<repo> --env <env> [--full] [--project <name>] [--environment <name>]
        cast capture   <org>/<repo> --env <env> [--path <dir>] [--project <name>] [--environment <name>] [--generated <NAME>] [--override <NAME>] [--force]
-       cast inventory <org>/<repo> --env <env> [--path <dir>] [--project <name>] [--environment <name>]
+       cast inventory <org>/<repo> --env <env> [--path <dir>] [--project <name>] [--environment <name>] [--resource <m>=<l>]
+       cast inventory --env <env> [--instance <name>]     # no repo: SWEEP the whole instance
        cast server add <name> --ip <ip> --key <file> --env <env> [--user root] [--port 22]
        cast smoke --env <env>
        cast team [--env <env>]
@@ -863,11 +871,55 @@ async function main(): Promise<number> {
     });
     const orgRepo = positionals[0];
     const envName = values.env;
-    if (!orgRepo || !envName) {
+    if (!envName) {
       console.error(USAGE);
       return 2;
     }
     const stateDir = stateDirFrom(values.state);
+    const sweepBindings = loadBindings(join(stateDir, "environments.yaml"));
+    const sweepBinding = sweepBindings.environments[envName];
+    if (!sweepBinding) {
+      console.error(`environment ${envName} not in environments.yaml`);
+      return 2;
+    }
+    // NO REPO → SWEEP. There is nothing to reconcile against, so don't pretend
+    // to: just show what is on the box. This is the pass that has to come first
+    // when the box is one you did not build, and requiring coordinates for it
+    // made inventory a discovery verb that needed you to have already
+    // discovered.
+    if (!orgRepo) {
+      const { instance, client } = openCoolify(
+        stateDir,
+        values.instance,
+        sweepBinding,
+      );
+      // The team assert matters MORE here than anywhere: Coolify scopes what a
+      // token can see to its team, so a wrong-team token sweeps an instance and
+      // truthfully reports that it is empty.
+      const team = await assertTeam(client, sweepBinding.team, envName);
+      console.log(`team ${formatTeam(team)} ✓`);
+      const projects: SweepProject[] = [];
+      for (const p of await client.projects()) {
+        const environments: SweepEnvironment[] = [];
+        for (const name of await client.environments(p.uuid)) {
+          const found = await fetchLive(client, p.name, name);
+          environments.push({
+            name,
+            resources: found.found
+              ? found.live.map((l) => ({ kind: l.kind, name: l.name }))
+              : [],
+          });
+        }
+        projects.push({ name: p.name, environments });
+      }
+      console.log(
+        renderSweep(projects, {
+          instance: instance.name,
+          baseUrl: instance.baseUrl,
+        }),
+      );
+      return 0;
+    }
     const repoShort = orgRepo.split("/")[1];
     const projectName = values.project ?? repoShort;
     const coolifyEnv = values.environment ?? envName;

src/coolify.ts

@@ -109,6 +109,44 @@ export class CoolifyClient {
     this.resolve("github app", "/github-apps", name);
   projectUuid = (name: string) => this.resolve("project", "/projects", name);
 
+  // Every project the TOKEN can see. Team-scoped by Coolify itself, which is
+  // why a sweep still asserts the team first: a wrong-team token sees nothing,
+  // and "nothing" would render as "this instance is empty".
+  async projects(): Promise<Array<{ uuid: string; name: string }>> {
+    return (await this.get("/projects")) as Array<{
+      uuid: string;
+      name: string;
+    }>;
+  }
+
+  // A project's environment NAMES.
+  //
+  // Two roads, because the vendored OpenAPI has been wrong before and this is a
+  // discovery path — the one place where failing to enumerate is worse than
+  // being slow. GET /projects/{uuid}/environments is documented; if a given
+  // instance does not serve it, GET /projects/{uuid} carries the same list as a
+  // relation on the project (ProjectController@show eager-loads it). Falling
+  // back beats reporting "no environments" for a project that has several.
+  async environments(projectUuid: string): Promise<string[]> {
+    const names = (items: unknown): string[] =>
+      Array.isArray(items)
+        ? items
+            .map((e) => (e as { name?: unknown })?.name)
+            .filter((n): n is string => typeof n === "string")
+        : [];
+    try {
+      const direct = await this.get(`/projects/${projectUuid}/environments`);
+      const found = names(direct);
+      if (found.length > 0) return found;
+    } catch (err) {
+      if (!(err instanceof HttpError) || err.status !== 404) throw err;
+    }
+    const project = (await this.get(`/projects/${projectUuid}`)) as {
+      environments?: unknown;
+    } | null;
+    return names(project?.environments);
+  }
+
   async deploy(uuid: string): Promise<void> {
     await this.post(`/deploy?uuid=${encodeURIComponent(uuid)}`);
   }

src/inventory.ts

@@ -54,6 +54,75 @@ export type Reconciliation = {
 
 const sorted = (xs: Iterable<string>) => [...xs].sort();
 
+// --- The sweep: what is on this box, before any manifest is involved ---
+//
+// The verb's premise is that you are looking at a box you did not build, so you
+// do NOT know its coordinates yet. Requiring a project and an environment to
+// look at it made it a discovery tool that needed you to have already
+// discovered — and the operator went back to hand-curling /projects to find out
+// where anything lived. This is that pass, and it needs no manifest at all.
+
+export type SweepEnvironment = {
+  name: string;
+  resources: Array<{ kind: string; name: string }>;
+};
+
+export type SweepProject = {
+  name: string;
+  environments: SweepEnvironment[];
+};
+
+export function renderSweep(
+  projects: SweepProject[],
+  ctx: { instance: string; baseUrl: string },
+): string {
+  const lines = [
+    `sweep — instance ${ctx.instance} (${ctx.baseUrl})`,
+    "",
+    "  Every project, every environment, every resource this token can see.",
+    "  No manifest involved: this is what is HERE, not how it compares to anything.",
+    "",
+  ];
+  if (projects.length === 0) {
+    lines.push(
+      "  (no projects at all)",
+      "",
+      "  An instance with no projects is more often a token that cannot see them",
+      "  than an empty instance — the team assert above is what rules that out.",
+    );
+    return lines.join("\n");
+  }
+  for (const p of projects) {
+    lines.push(`  ${p.name}`);
+    if (p.environments.length === 0) {
+      lines.push("    (no environments)");
+    }
+    for (const e of p.environments) {
+      const counts = Object.entries(
+        e.resources.reduce<Record<string, number>>((acc, r) => {
+          acc[r.kind] = (acc[r.kind] ?? 0) + 1;
+          return acc;
+        }, {}),
+      )
+        .map(([kind, n]) => `${n} ${kind}${n === 1 ? "" : "s"}`)
+        .join(", ");
+      // An empty environment is worth seeing, not hiding: Coolify auto-creates
+      // `production` in every project, and an operator who assumes that is where
+      // things live will aim every later command at nothing.
+      lines.push(`    ${e.name.padEnd(14)} ${counts || "(empty)"}`);
+      for (const r of e.resources) {
+        lines.push(`      ${r.kind.padEnd(12)} ${r.name}`);
+      }
+    }
+    lines.push("");
+  }
+  lines.push(
+    "Point a reconciliation at one of these with --project / --environment, and",
+    "map any resource whose name differs with --resource <manifest>=<live>.",
+  );
+  return lines.join("\n");
+}
+
 export function reconcile(
   manifest: ManifestResource[],
   live: LiveResource[],
@@ -98,6 +167,38 @@ export function renderInventory(
     environment: string;
   },
 ): string {
+  // The box has NOTHING in it. Not "nothing that matched" — nothing at all.
+  //
+  // This is the third face of the D-237 lie, and the quietest: an environment
+  // with zero resources reads back exactly like a box you have not built yet,
+  // and the report then consists entirely of the manifest talking to itself.
+  // The overall impression — "the box has nothing, the manifest has five things"
+  // — is how a full-create plan gets laundered into a pass. It happened: pointed
+  // at a project's auto-created `production` environment, this verb reported
+  // five differences against a box whose resources were alive and serving
+  // production the whole time, in an environment named `staging`.
+  if (rec.matched.length === 0 && rec.boxOnly.length === 0) {
+    return [
+      `inventory — ${ctx.orgRepo} ${ctx.env}`,
+      "",
+      `  looked in:    project "${ctx.project}", environment "${ctx.environment}"`,
+      "  found:        NOTHING. Not one resource.",
+      "",
+      "This environment is EMPTY — so there is nothing here to reconcile, and the",
+      "manifest's list below would just be the manifest talking to itself.",
+      "",
+      "An environment with zero resources is far more often the WRONG COORDINATE",
+      "than an empty one. Coolify auto-creates a `production` environment in every",
+      "project, and a box built by hand keeps its real resources wherever someone",
+      "put them — which may well be an environment called something else entirely.",
+      "",
+      "Sweep the instance and see where things actually are:",
+      "",
+      `    cast inventory --env ${ctx.env} --instance ${ctx.instance}`,
+      "",
+      `(The manifest declares: ${rec.manifestOnly.map((m) => m.name).join(", ")}.)`,
+    ].join("\n");
+  }
   const lines = [
     `inventory — ${ctx.orgRepo} ${ctx.env}`,
     "",

test/sweep-cli.test.ts

@@ -0,0 +1,212 @@
+import { spawn } from "node:child_process";
+import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
+import { createServer } from "node:http";
+import type { AddressInfo } from "node:net";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+
+// The sweep, against a stub shaped like the box that made it necessary.
+//
+// Three projects — one ours, two unrelated third-party client sites nobody knew
+// were there. Our project has TWO environments: `production`, which Coolify
+// auto-created and which is EMPTY, and `staging`, which is where the live
+// founder-facing system has been running the whole time because nobody ever
+// swapped it.
+//
+// Pointed at `production`, the old inventory reported "5 differences" against a
+// box whose resources were alive and serving production — the manifest talking
+// to itself, and an impression ("the box has nothing") that is exactly how a
+// full-create plan gets laundered into a pass.
+
+type Stub = { url: string; close: () => Promise<void> };
+const stubs: Stub[] = [];
+
+async function stubCoolify(): Promise<Stub> {
+  const server = createServer((req, res) => {
+    const path = (req.url ?? "").replace("/api/v1", "");
+    const json = (body: unknown) => {
+      res.writeHead(200, { "content-type": "application/json" });
+      res.end(JSON.stringify(body));
+    };
+    if (path === "/teams/current") return json({ id: 0, name: "Root Team" });
+    if (path === "/projects")
+      return json([
+        { uuid: "p1", name: "Incubator" },
+        { uuid: "p2", name: "La Familia Site" },
+        { uuid: "p3", name: "Martin Reyes Barber Shop" },
+      ]);
+    if (path === "/projects/p1/environments")
+      return json([{ name: "production" }, { name: "staging" }]);
+    if (path === "/projects/p2/environments")
+      return json([{ name: "production" }]);
+    if (path === "/projects/p3/environments")
+      return json([{ name: "production" }]);
+    // Coolify auto-creates `production`. It is empty. Everything real lives in
+    // `staging`, under names a human typed.
+    if (path === "/projects/p1/production") return json({});
+    if (path === "/projects/p1/staging")
+      return json({
+        applications: [
+          { name: "Incubator Stack v2", uuid: "a1" },
+          { name: "Incubator Landing", uuid: "a2" },
+        ],
+        postgresqls: [{ name: "Incubator Database v2", uuid: "d1" }],
+        services: [{ name: "Incubator Umami", uuid: "s1" }],
+      });
+    if (path === "/projects/p2/production")
+      return json({ applications: [{ name: "lafamilia-web", uuid: "a9" }] });
+    if (path === "/projects/p3/production")
+      return json({ applications: [{ name: "barber-web", uuid: "a8" }] });
+    if (path.endsWith("/envs")) return json([]);
+    res.writeHead(404);
+    res.end("{}");
+  });
+  await new Promise<void>((r) => {
+    server.listen(0, "127.0.0.1", r);
+  });
+  const stub: Stub = {
+    url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`,
+    close: () =>
+      new Promise<void>((r) => {
+        server.close(() => r());
+      }),
+  };
+  stubs.push(stub);
+  return stub;
+}
+
+afterEach(async () => {
+  await Promise.all(stubs.splice(0).map((s) => s.close()));
+});
+
+const MANIFEST = `project: incubator
+environments:
+  staging:
+    applications:
+      core:
+        source: { repo: heavy-duty/incubator, branch: main }
+        build: { pack: nixpacks, base_directory: / }
+        domains: ["http://core.example.com"]
+`;
+
+function fixture(url: string) {
+  const checkout = mkdtempSync(join(tmpdir(), "cast-co-"));
+  mkdirSync(join(checkout, ".infra", "env"), { recursive: true });
+  writeFileSync(join(checkout, ".infra", "manifest.yaml"), MANIFEST);
+  const state = mkdtempSync(join(tmpdir(), "cast-state-"));
+  mkdirSync(join(state, "secrets"));
+  writeFileSync(
+    join(state, ".coolify.env"),
+    `COOLIFY_BASE_URL="${url}"\nCOOLIFY_ACCESS_TOKEN="t"\n`,
+  );
+  writeFileSync(
+    join(state, "environments.yaml"),
+    [
+      "environments:",
+      "  staging:",
+      "    server: staging-box",
+      "    team: { id: 0, name: Root Team }",
+      "github_apps:",
+      "  incubator: hdb-coolify",
+      "",
+    ].join("\n"),
+  );
+  return { checkout, state };
+}
+
+function run(args: string[]): Promise<{ code: number; output: string }> {
+  return new Promise((resolve) => {
+    const child = spawn("node", ["dist/cli.js", "inventory", ...args], {
+      stdio: ["pipe", "pipe", "pipe"],
+    });
+    let output = "";
+    child.stdout.on("data", (d) => {
+      output += String(d);
+    });
+    child.stderr.on("data", (d) => {
+      output += String(d);
+    });
+    child.stdin.end();
+    child.on("close", (code) => resolve({ code: code ?? 0, output }));
+  });
+}
+
+describe("cast inventory — the sweep (#22)", () => {
+  it("with no repo, shows every project, environment and resource", async () => {
+    const f = fixture((await stubCoolify()).url);
+    const r = await run(["--env", "staging", "--state", f.state]);
+    expect(r.code).toBe(0);
+
+    // Every project the token can see — including the two nobody knew were on
+    // the box. (On the real instance, that discovery is what stopped a plan
+    // whose next step would have deleted the box.)
+    expect(r.output).toContain("Incubator");
+    expect(r.output).toContain("La Familia Site");
+    expect(r.output).toContain("Martin Reyes Barber Shop");
+
+    // Both environments, and the empty one is SHOWN, not hidden — an operator
+    // who assumes `production` is where things live aims every later command at
+    // nothing.
+    expect(r.output).toContain("production");
+    expect(r.output).toContain("(empty)");
+    expect(r.output).toContain("staging");
+
+    // And the resources, under the names the box actually uses.
+    expect(r.output).toContain("Incubator Stack v2");
+    expect(r.output).toContain("Incubator Database v2");
+  });
+
+  it("needs no repo, no manifest, no store — it runs before adoption exists", async () => {
+    const f = fixture((await stubCoolify()).url);
+    // No org/repo positional at all: nothing is cloned, no manifest is read.
+    const r = await run(["--env", "staging", "--state", f.state]);
+    expect(r.code).toBe(0);
+    expect(r.output).toContain("No manifest involved");
+  });
+
+  it("still asserts the team — a wrong-team token would sweep an empty instance", async () => {
+    const f = fixture((await stubCoolify()).url);
+    writeFileSync(
+      join(f.state, "environments.yaml"),
+      [
+        "environments:",
+        "  staging:",
+        "    server: staging-box",
+        "    team: { id: 9, name: Some Other Team }",
+        "",
+      ].join("\n"),
+    );
+    const r = await run(["--env", "staging", "--state", f.state]);
+    // Coolify scopes what a token can see to its team, so an unasserted sweep
+    // would truthfully report that the instance is empty.
+    expect(r.code).not.toBe(0);
+    expect(r.output).not.toContain("La Familia Site");
+  });
+});
+
+describe("an empty environment shouts (#22)", () => {
+  it("says NOTHING is here, and names the sweep — not '5 differences'", async () => {
+    const f = fixture((await stubCoolify()).url);
+    const r = await run([
+      "heavy-duty/incubator",
+      "--env",
+      "staging",
+      "--state",
+      f.state,
+      "--path",
+      f.checkout,
+      "--project",
+      "Incubator",
+      "--environment",
+      "production", // auto-created by Coolify, and empty
+    ]);
+    expect(r.code).toBe(0);
+    expect(r.output).toContain("NOTHING");
+    expect(r.output).toContain("WRONG COORDINATE");
+    expect(r.output).toContain("cast inventory --env staging");
+    // NOT the old output, whose overall impression was "the box has nothing and
+    // the manifest has five things" — a full-create plan in all but name.
+    expect(r.output).not.toContain("difference(s) between the manifest");
+  });
+});