From 276c9bd2a751f90911689a37e3a86f4a26056e23 Mon Sep 17 00:00:00 2001 From: Kumar Damani Date: Tue, 15 Sep 2026 16:29:29 -0400 Subject: automate adding new host flow --- manage.sh | 531 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 531 insertions(+) create mode 100755 manage.sh (limited to 'manage.sh') diff --git a/manage.sh b/manage.sh new file mode 100755 index 0000000..f89f11a --- /dev/null +++ b/manage.sh @@ -0,0 +1,531 @@ +#!/usr/bin/env bash +# Top-level entry point for management. +# Run without args for the interactive menu. +set -euo pipefail + +# Resolve paths relative to this script, not the caller's CWD +cd -- "$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")" + +die() { echo "❌ $*" >&2; exit 1; } + +usage() { + cat <<'EOF' +usage: manage.sh # interactive menu + manage.sh add [flags] + manage.sh update [switch|boot] + manage.sh update-all + manage.sh base # rebuild base image, restore + template on proxmox + +add flags: + --cpu N cores (default 1) + --mem MB memory in MB (default 1024) + --disk SIZE e.g. 10G (default 10G) + --vlan TAG network tag (default 30) + --tf show plan, ask for approval, then tofu/terraform apply + --ssh wait for ssh to come up + --deploy nix/deploy.sh jefe + --all shorthand for --tf --ssh --deploy + --yes non-interactive: auto-approve the plan, skip prompts +EOF +} + +hosts() { ls nix/per-host | grep -v '^base$'; } + +confirm() { + local r="" + read -rp "$1 [y/N]: " r || r="" + [[ $r =~ ^[Yy]$ ]] +} + +# Map a numeric selection (or a literal name) to a host name +resolve_host() { + local input="$1" list=() + readarray -t list < <(hosts) + if [[ $input =~ ^[0-9]+$ ]]; then + (( input >= 1 && input <= ${#list[@]} )) || die "invalid selection: $input (pick 1..${#list[@]})" + printf '%s' "${list[input-1]}" + else + printf '%s' "$input" + fi +} + +pick_host() { + local host="${1:-}" + if [[ -z $host ]]; then + local list=() i + readarray -t list < <(hosts) + { + echo "Hosts:" + for i in "${!list[@]}"; do printf ' %2d) %s\n' "$((i+1))" "${list[i]}"; done + } >&2 + read -rp "Host name/number: " input || die "aborted" + [[ -n $input ]] || die "aborted" + fi + host=$(resolve_host "${host:-$input}") + [[ -d "nix/per-host/$host" ]] || die "no such host: '$host' (expected nix/per-host/$host)" + printf '%s' "$host" +} + +# ---- actions ---- + +cmd_update_host() { + local host target + host=$(pick_host "${1:-}") + target="${2:-}" + if [[ -z $target ]]; then + read -rp "Deploy action [switch/boot, default switch]: " target || target="" + target="${target:-switch}" + fi + [[ $target == switch || $target == boot ]] || die "action must be 'switch' or 'boot'" + nix/deploy.sh "$host" jefe "$target" +} + +cmd_update_all() { + echo "→ updating flake inputs ..." + (cd nix && nix flake update) + local host + for host in $(hosts); do + if ! ssh -o ConnectTimeout=5 "$host" true 2>/dev/null; then + echo "→ [$host] offline, skipping" + continue + fi + # `boot` avoids the auto-switch restriction; the reboot activates it. + if nix/deploy.sh "$host" jefe boot; then + ssh "$host" sudo reboot || true + else + echo "→ [$host] deploy failed, not rebooting" + fi + done +} + +cmd_update_base() { + echo "→ building base image ..." + (cd nix && nix flake update && nixos-rebuild build-image --image-variant proxmox --flake .#base) + local img + img=$(ls -t nix/result/*.vma.zst 2>/dev/null | head -n1) || die "no image found in nix/result/" + echo "→ image: $(basename "$img")" + echo "→ copying to lan.hesh:/var/lib/vz/dump/ ..." + scp "$img" lan.hesh:/var/lib/vz/dump/ + + # 104 is the clone source in TF (full clones, so no linked-clone breakage). + confirm "→ Destroy template 104 and restore from this image?" || { + echo "→ stopped after copy; restore manually" + return 0 + } + ssh lan.hesh 'qm destroy 104 --purge' + ssh lan.hesh "qmrestore /var/lib/vz/dump/$(basename "$img") 104 --storage local-lvm" + ssh lan.hesh 'qm template 104' + echo "✅ template 104 updated" +} + +# ---- add a new host (scaffold + optional full flow) ---- + +cmd_add() { + local HOST="" CPU=1 MEM=1024 DISK="10G" VLAN=30 + local CPU_GIVEN=0 MEM_GIVEN=0 DISK_GIVEN=0 VLAN_GIVEN=0 + local RUN_TF=0 RUN_SSH=0 RUN_DEPLOY=0 AUTO_YES=0 + + # Interactive menu: offer the full flow up front + if (( $# == 0 )) && [[ -t 0 ]]; then + local r="" + read -rp "→ Run full flow (tofu apply, ssh wait, deploy)? [Y/n]: " r || r="" + [[ $r =~ ^[Nn]$ ]] || { RUN_TF=1; RUN_SSH=1; RUN_DEPLOY=1; } + fi + + while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) usage; return 0 ;; + --tf) RUN_TF=1 ;; + --ssh) RUN_SSH=1 ;; + --deploy) RUN_DEPLOY=1 ;; + --all) RUN_TF=1; RUN_SSH=1; RUN_DEPLOY=1 ;; + -y|--yes) AUTO_YES=1 ;; + --cpu) [[ $# -ge 2 ]] || die "--cpu needs a value"; CPU=$2; CPU_GIVEN=1; shift ;; + --mem) [[ $# -ge 2 ]] || die "--mem needs a value"; MEM=$2; MEM_GIVEN=1; shift ;; + --disk) [[ $# -ge 2 ]] || die "--disk needs a value"; DISK=$2; DISK_GIVEN=1; shift ;; + --vlan) [[ $# -ge 2 ]] || die "--vlan needs a value"; VLAN=$2; VLAN_GIVEN=1; shift ;; + -*) usage >&2; die "unknown flag: $1" ;; + *) [[ -z $HOST ]] || die "unexpected argument: $1"; HOST=$1 ;; + esac + shift + done + + if [[ -z $HOST ]]; then + read -rp "Host name: " HOST + fi + [[ $HOST =~ ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ ]] || die "invalid host name: '$HOST'" + [[ -d nix/per-host/$HOST && $HOST != base ]] && echo "→ note: nix/per-host/$HOST already exists" + + [[ $CPU =~ ^[0-9]+$ ]] || die "--cpu must be an integer" + [[ $MEM =~ ^[0-9]+$ ]] || die "--mem must be an integer (MB)" + [[ $VLAN =~ ^[0-9]+$ && $VLAN -le 4094 ]] || die "--vlan must be 0..4094" + [[ $DISK =~ ^[0-9]+[KMGT]B?$ ]] || die "--disk must look like 10G / 100G / 512M" + + # Prompt (interactive only) for anything not given; defaults come from monitoring-2. + if [[ -t 0 ]] && ! (( AUTO_YES )); then + (( CPU_GIVEN )) || { read -rp "CPU cores [$CPU]: " r || r=""; CPU=${r:-$CPU}; } + (( MEM_GIVEN )) || { read -rp "Memory MB [$MEM]: " r || r=""; MEM=${r:-$MEM}; } + (( DISK_GIVEN )) || { read -rp "Disk size [$DISK]: " r || r=""; DISK=${r:-$DISK}; } + (( VLAN_GIVEN )) || { read -rp "VLAN tag [$VLAN]: " r || r=""; VLAN=${r:-$VLAN}; } + fi + + local SNAKE="${HOST//-/_}" + local SOPS_PREFIX + SOPS_PREFIX=$(sed -E 's/-[0-9]+$//' <<<"$HOST") # torrents-2 -> torrents (for path_regex) + + # ---- host keys + age key ---- + if [[ -f "nix/host-keys/$HOST/ssh_host_ed25519_key.pub" ]]; then + echo "→ [$HOST] host keys exist, skipping generation" + else + nix/generate-keys.sh "$HOST" >/dev/null + echo "→ [$HOST] generated host keys" + fi + + if ! command -v ssh-to-age >/dev/null; then + die "ssh-to-age not found (direnv should provide it via nix/.envrc)" + fi + local AGE_KEY + AGE_KEY=$(ssh-to-age < "nix/host-keys/$HOST/ssh_host_ed25519_key.pub") + [[ $AGE_KEY == age1* ]] || die "failed to derive age key for $HOST" + + # ---- .sops.yaml: key anchor ---- + if grep -q "&server_${SNAKE} " nix/.sops.yaml; then + echo "→ [$HOST] .sops.yaml anchor exists, skipping" + else + awk -v line=" - &server_${SNAKE} ${AGE_KEY}" ' + !done && /^creation_rules:/ { print line; done=1 } + { print }' nix/.sops.yaml > nix/.sops.yaml.tmp && mv nix/.sops.yaml.tmp nix/.sops.yaml + echo "→ [$HOST] added age key to .sops.yaml" + fi + + # ---- .sops.yaml: creation rule (extend an existing prefix rule if present) ---- + if grep -qE "^ +- \*server_${SNAKE}\$" nix/.sops.yaml; then + echo "→ [$HOST] .sops.yaml creation rule exists, skipping" + elif grep -q "path_regex: .secrets/${SOPS_PREFIX}\." nix/.sops.yaml; then + awk -v prefix="$SOPS_PREFIX" -v newline=" - *server_${SNAKE}" ' + /^ - path_regex/ { inrule=0 } + $0 ~ (" - path_regex: .secrets/" prefix) { inrule=1 } + inrule && /^ - / { last=NR } + /^$/ { inrule=0 } + { lines[NR]=$0 } + END { + for (i=1; i<=NR; i++) { + print lines[i] + if (i == last) print newline + } + }' nix/.sops.yaml > nix/.sops.yaml.tmp && mv nix/.sops.yaml.tmp nix/.sops.yaml + echo "→ [$HOST] extended existing creation rule for 'secrets/${SOPS_PREFIX}.*'" + else + cat >> nix/.sops.yaml < "nix/per-host/$HOST/configuration.nix" <=1; i--) { + if (lines[i] == " };" && (i == NR || lines[i+1] == " };")) { ins=i; break } + } + for (i=1; i<=NR; i++) { + if (i == ins) { print block; print "" } + print lines[i] + } + }' nix/flake.nix > nix/flake.nix.tmp && mv nix/flake.nix.tmp nix/flake.nix + echo "→ [$HOST] added nixosConfigurations entry to flake.nix" + fi + + # ---- secrets ---- + if [[ -f "nix/secrets/$HOST.yaml" ]]; then + echo "→ [$HOST] nix/secrets/$HOST.yaml exists, skipping" + else + command -v sops >/dev/null || die "sops not found (direnv should provide it via nix/.envrc)" + printf 'foo: bar\n' > "nix/secrets/$HOST.yaml" + (cd nix && sops -e -i "secrets/$HOST.yaml") || die "sops encrypt failed" + echo "→ [$HOST] created nix/secrets/$HOST.yaml (foo: bar)" + fi + + # ---- Terraform VM (only if it doesn't exist) ---- + local TF_CREATED=0 + if grep -q "proxmox_vm_qemu\" \"${SNAKE}\"" terraform/main.tf; then + echo "→ [$HOST] TF resource proxmox_vm_qemu.${SNAKE} exists, skipping" + else + local ORDER + ORDER=$(grep -E '^\s*order\s*=' terraform/main.tf \ + | grep -oE '[0-9]+' | sort -n | tail -n1) + ORDER=$(( ${ORDER:-0} + 1 )) + cat >> terraform/main.tf <> "$SSH_CONFIG" <> nix/README.md <> nix/README.md + fi + echo "→ [$HOST] added README Hosts section" + fi + + # ---- Phase: tofu plan/approve/apply (idempotent; plan is shown before apply) ---- + local TF_APPLIED=0 TF_CLEAN=0 TF_BLOCKED=0 + if (( RUN_TF )); then + local TOFU=() + if command -v tofu >/dev/null; then TOFU=(tofu) + elif command -v terraform >/dev/null; then TOFU=(terraform) + else + die "tofu/terraform not on PATH (direnv provides it via nix/.envrc)" + fi + echo "→ [$HOST] running ${TOFU[0]} plan ..." + rm -f terraform/tfplan # clear any stale plan + local PLAN_OUT + PLAN_OUT=$( cd terraform && + # shellcheck disable=SC1091 + source con.env && "${TOFU[@]}" plan -out=tfplan ) || die "tofu plan failed" + printf '%s\n' "$PLAN_OUT" + + if grep -q "No changes." <<<"$PLAN_OUT"; then + echo "→ [$HOST] no infrastructure changes; apply not needed" + TF_CLEAN=1 + else + local approved=0 + if (( AUTO_YES )); then + approved=1 + elif [[ -t 0 ]]; then + read -rp "→ Apply the above plan? [y/N]: " r || r="" + if [[ $r =~ ^[Yy]$ ]]; then approved=1; fi + else + echo "→ [$HOST] no tty available for approval; use --yes to auto-approve" + fi + if (( approved )); then + echo "→ [$HOST] applying ..." + ( cd terraform && + # shellcheck disable=SC1091 + source con.env && "${TOFU[@]}" apply tfplan ) || die "tofu apply failed" + echo "→ [$HOST] apply done" + TF_APPLIED=1 + else + echo "→ [$HOST] apply declined" + TF_BLOCKED=1 + fi + fi + rm -f terraform/tfplan + fi + + # ---- Phase: wait for ssh (VM first boots as 'nixos' via the ~/.ssh/config workaround) ---- + if (( RUN_SSH && TF_BLOCKED )); then + echo "→ [$HOST] skipping ssh phase (apply was declined)" + elif (( RUN_SSH )); then + local SSH_TRIES=30 SSH_WAIT=5 ok=0 + echo "→ [$HOST] waiting for ssh (up to $((SSH_TRIES * SSH_WAIT))s) ..." + local i + for i in $(seq 1 "$SSH_TRIES"); do + if ssh -o ConnectTimeout=5 "$HOST" true 2>/dev/null; then ok=1; break; fi + sleep "$SSH_WAIT" + done + (( ok )) || die "ssh to $HOST failed after $SSH_TRIES attempts" + echo "→ [$HOST] ssh ok" + fi + + # ---- Phase: deploy (nixos-rebuild switch is idempotent) ---- + if (( RUN_DEPLOY && TF_BLOCKED )); then + echo "→ [$HOST] skipping deploy phase (apply was declined)" + elif (( RUN_DEPLOY )); then + echo "→ [$HOST] deploying ..." + nix/deploy.sh "$HOST" jefe + echo "→ [$HOST] deploy done" + fi + + # ---- Summary ---- + echo "" + echo "================ ${HOST} ================" + echo " age key: ${AGE_KEY}" + echo " sops prefix: secrets/${SOPS_PREFIX}.*.yaml" + echo " flake target: .#${HOST}" + echo " per-host config: nix/per-host/${HOST}/configuration.nix" + echo " secrets: nix/secrets/${HOST}.yaml (foo: bar)" + if (( TF_CREATED )); then + echo " TF resource: proxmox_vm_qemu.${SNAKE} (created: cpu=${CPU} mem=${MEM} disk=${DISK} vlan=${VLAN})" + else + echo " TF resource: proxmox_vm_qemu.${SNAKE} (already in TF)" + fi + echo "" + echo "Next steps:" + if (( TF_CREATED && ! RUN_TF )) || (( RUN_TF && ! TF_APPLIED && ! TF_CLEAN )); then + cat < " choice || exit 0 + case "$choice" in + # subshells: a die/validation failure inside an action must not kill the menu + 1) ( cmd_add ) || echo "✗ add failed" ;; + 2) ( cmd_update_host ) || echo "✗ update failed" ;; + 3) ( cmd_update_all ) || echo "✗ update-all failed" ;; + 4) ( cmd_update_base ) || echo "✗ base update failed" ;; + q|quit) exit 0 ;; + "") ;; + *) echo "? unknown choice: $choice" ;; + esac + done +} + +case "${1:-}" in + "" ) menu ;; + -h|--help|help) usage ;; + add) shift; cmd_add "$@" ;; + update) shift; cmd_update_host "$@" ;; + update-all) cmd_update_all ;; + base) cmd_update_base ;; + *) usage >&2; die "unknown command: $1" ;; +esac -- cgit v1.2.3