'incus admin init --minimal' picks the 'dir' backend, which has no copy-on-write: every snapshot and clone is a full copy of the box's root — several GB and minutes apiece once Docker, Node and Claude Code are installed. Measured live: one clone took minutes, and a single drill run pays that three times (snapshot + two clones). The workflow the tool exists for — log in once, snapshot, clone forever — stops being attractive at exactly that price. setup-host.sh now bootstraps via a preseed that mirrors what --minimal creates (default pool, incusbr0, default profile with root + eth0) with only the driver deliberate: btrfs on a loop device (CoW — clones share blocks and diverge on write), installing btrfs-progs if absent, falling back to dir with a warning if btrfs cannot be had. Existing hosts are untouched: the block is skipped whenever a 'default' pool exists. Closes #29
147 lines
7.1 KiB
Bash
Executable file
147 lines
7.1 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# One-time host setup: install Incus, create the isolated network + ACL and
|
|
# the claude-dev profile. Idempotent. Ubuntu 24.04 / Debian 13.
|
|
set -euo pipefail
|
|
|
|
here="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
|
|
if ! command -v incus >/dev/null; then
|
|
sudo apt-get update
|
|
sudo apt-get install -y incus
|
|
fi
|
|
|
|
if ! id -nG "$USER" | grep -qw incus-admin; then
|
|
sudo usermod -aG incus-admin "$USER"
|
|
echo "NOTE: added $USER to incus-admin — re-login (or 'sg incus-admin') and re-run."
|
|
exit 0
|
|
fi
|
|
|
|
# Storage pool + base config (safe to re-run: skipped once the pool exists).
|
|
# NOT 'incus admin init --minimal': minimal picks the 'dir' backend, which has
|
|
# no copy-on-write — every snapshot and clone is a FULL copy of the box's root,
|
|
# several GB and minutes apiece once a box is provisioned, against a workflow
|
|
# whose whole point is "log in once, snapshot, clone forever" (#29). btrfs on
|
|
# a loop device gives CoW (near-instant, near-free clones) with no
|
|
# partitioning. The preseed mirrors exactly what --minimal creates (pool,
|
|
# incusbr0, default profile) with only the driver deliberate; dir remains the
|
|
# fallback so a host that cannot do btrfs still works — just slowly, and it
|
|
# says so.
|
|
if ! incus storage show default >/dev/null 2>&1; then
|
|
driver=btrfs
|
|
command -v mkfs.btrfs >/dev/null 2>&1 || sudo apt-get install -y btrfs-progs || driver=dir
|
|
if ! incus admin init --preseed <<PRESEED
|
|
storage_pools:
|
|
- name: default
|
|
driver: $driver
|
|
networks:
|
|
- name: incusbr0
|
|
type: bridge
|
|
profiles:
|
|
- name: default
|
|
devices:
|
|
root:
|
|
path: /
|
|
pool: default
|
|
type: disk
|
|
eth0:
|
|
name: eth0
|
|
network: incusbr0
|
|
type: nic
|
|
PRESEED
|
|
then
|
|
echo "storage: $driver preseed failed — falling back to --minimal (dir: every clone is a full disk copy)" >&2
|
|
incus admin init --minimal
|
|
fi
|
|
echo "storage: pool 'default' driver = $(incus storage show default | awk '/^driver:/ {print $2}')"
|
|
fi
|
|
|
|
# Isolated NAT network. IPv6 off: one less egress path to reason about.
|
|
incus network show claudenet >/dev/null 2>&1 || incus network create claudenet \
|
|
ipv4.address=10.87.0.1/24 ipv4.nat=true ipv6.address=none
|
|
|
|
# ACL: default egress allow (internet), explicit drops for private space.
|
|
# Gateway carve-out first so instance DNS (dnsmasq on 10.87.0.1) survives.
|
|
if ! incus network acl show claude-isolate >/dev/null 2>&1; then
|
|
incus network acl create claude-isolate
|
|
incus network acl rule add claude-isolate egress action=allow destination=10.87.0.1/32
|
|
incus network acl rule add claude-isolate egress action=drop destination=10.0.0.0/8
|
|
incus network acl rule add claude-isolate egress action=drop destination=172.16.0.0/12
|
|
incus network acl rule add claude-isolate egress action=drop destination=192.168.0.0/16
|
|
incus network acl rule add claude-isolate egress action=drop destination=169.254.0.0/16
|
|
incus network acl rule add claude-isolate egress action=drop destination=100.64.0.0/10
|
|
fi
|
|
incus network set claudenet security.acls=claude-isolate \
|
|
security.acls.default.egress.action=allow \
|
|
security.acls.default.ingress.action=drop
|
|
|
|
# A box must not be able to ENUMERATE its siblings, either. dnsmasq on the
|
|
# gateway serves DNS (that carve-out is what makes egress resolution work) and
|
|
# it holds a record for every instance on the network — so 'getent hosts <box>'
|
|
# from inside one box resolved another's name and address. Connection blocked,
|
|
# reconnaissance wide open. dns.mode=none stops it registering instance records;
|
|
# forwarding for public names is unaffected (verified live).
|
|
incus network set claudenet dns.mode=none
|
|
|
|
# A box's resolver must not be a function of the host's VPN posture (#33).
|
|
# The bridge's dnsmasq forwards to whatever sits in the HOST's /etc/resolv.conf
|
|
# at that moment. On a Tailscale/VPN host that is MagicDNS: box DNS flaps with
|
|
# the tailnet (this is what killed cold mints), and tailnet peer names and
|
|
# split-DNS zones RESOLVE from inside a box — name-level reconnaissance of a
|
|
# private network, the same shape as the sibling enumeration closed above.
|
|
# no-resolv detaches dnsmasq from the host's resolver entirely; server= pins a
|
|
# stable public upstream (override: BOX_DNS="ip ip…"). raw.dnsmasq is the
|
|
# lever — the bridge has no first-class upstream key. Verified live on the
|
|
# drill host: pin applied, box resolves, cold mint survives.
|
|
BOX_DNS="${BOX_DNS:-1.1.1.1 8.8.8.8}"
|
|
incus network set claudenet raw.dnsmasq \
|
|
"$(printf 'no-resolv\n'; for s in $BOX_DNS; do printf 'server=%s\n' "$s"; done)"
|
|
|
|
# Sibling isolation itself is NOT an ACL rule — an L3 ACL never sees frames
|
|
# switched between two ports of one bridge. It lives in claudebox-firewall.sh
|
|
# as an nftables bridge-family rule. See the comment there; it is the reason
|
|
# boxes cannot reach each other.
|
|
|
|
# IPv6 stays off (ipv6.address=none, above). Every rule in the ACL and every
|
|
# rule in the firewall is IPv4-only, so IPv6 would be an uncovered path, not a
|
|
# feature. That is a contract, not a default.
|
|
|
|
# --- Firewall coexistence ---------------------------------------------------
|
|
# Hosts running UFW (INPUT drop) and/or Docker (FORWARD drop) silently eat
|
|
# claudenet traffic. Punch minimal, ordered holes; the Incus ACL still layers
|
|
# on top. The trailing deny also blocks instance -> host's own (public) IPs,
|
|
# which the RFC1918-only ACL cannot express. Rules live in
|
|
# claudebox-firewall.sh; a boot-time systemd unit re-applies the runtime-only
|
|
# parts (nft table, DOCKER-USER) after every reboot.
|
|
# The no-UFW path drives nft directly, and a stock Debian 13 cloud image ships
|
|
# neither nftables nor UFW — install the dependency we are about to use.
|
|
if ! command -v ufw >/dev/null 2>&1 && ! command -v nft >/dev/null 2>&1; then
|
|
sudo apt-get install -y nftables
|
|
fi
|
|
sudo install -m 755 "$here/host/claudebox-firewall.sh" /usr/local/sbin/claudebox-firewall
|
|
sudo install -m 644 "$here/host/claudebox-firewall.service" /etc/systemd/system/
|
|
sudo systemctl daemon-reload
|
|
sudo systemctl enable claudebox-firewall.service
|
|
# RESTART, not 'enable --now'. The unit is RemainAfterExit, so once it has run
|
|
# it stays "active" forever — and 'enable --now' does nothing to an active unit.
|
|
# Re-running setup-host after upgrading claudebox therefore installed the new
|
|
# rules to /usr/local/sbin and never applied them: the host kept the old
|
|
# firewall, silently, and the box→box hole stayed open through a release that
|
|
# claimed to close it. Restart re-runs the script, which is idempotent by design.
|
|
sudo systemctl restart claudebox-firewall.service
|
|
|
|
# Profile
|
|
if ! incus profile show claude-dev >/dev/null 2>&1; then
|
|
incus profile create claude-dev
|
|
fi
|
|
incus profile edit claude-dev < "$here/profiles/claude-dev.yaml"
|
|
|
|
# The sibling drop is the one rule whose absence is invisible: everything keeps
|
|
# working, and boxes can simply reach each other. Assert it landed.
|
|
if sudo nft list table bridge claudebox >/dev/null 2>&1; then
|
|
echo "Isolation: box-to-box drop is live (nft bridge table 'claudebox')."
|
|
else
|
|
echo "WARNING: the box-to-box drop is NOT active — boxes can reach each other." >&2
|
|
echo " check: sudo /usr/local/sbin/claudebox-firewall ; sudo nft list table bridge claudebox" >&2
|
|
fi
|
|
|
|
echo "Host ready. Launch with: claudebox new --name <box>"
|