monitoring config

6 files changed, 234 insertions, 3 deletions

diff --git a/nix/.sops.yaml b/nix/.sops.yaml
index b657a25..cf20e88 100644
--- a/nix/.sops.yaml
+++ b/nix/.sops.yaml
@@ -7,6 +7,7 @@ keys:
   - &server_photos age1zdy4saxtcs6y90mrjlmz6hnknk67w0cd3n79g2y6fvtzzfsq6f6shwamzj
   - &server_caldav age1mhugjsg0pz45p56p5vc6hz05zrnk630j6p5xu4n2nsskvtxg8cqsndqr90
   - &server_s3 age1nlkl8h7x33ssq3m5ka9syxurxz37y2duzhkvx3mmv23fxhetkuksqap73n
+  - &server_monitoring_2 age177u6hs3funxd8eqd4h0krynrtn2et74m6dk2pk0q7rezw3w23dpsjtytte
 
 creation_rules:
   - path_regex: 'secrets/torrents.*\.yaml$'
@@ -50,3 +51,10 @@ creation_rules:
           - *admin_me
           - *admin_me_2
           - *server_s3
+
+  - path_regex: 'secrets/monitoring.*\.yaml$'
+    key_groups:
+      - age:
+          - *admin_me
+          - *admin_me_2
+          - *server_monitoring_2
diff --git a/nix/README.md b/nix/README.md
index ab3a339..95cd4ac 100644
--- a/nix/README.md
+++ b/nix/README.md
@@ -38,15 +38,27 @@ tofu apply tfplan
 
 ## Add Host
 
-Copy `per-host/<EXISTING_HOST>` to `per-host/<HOST_NAME>`.
+Scaffold a new host (host keys, `.sops.yaml`, `flake.nix`, `per-host/`, secrets, TF VM if absent, `~/.ssh/config`, Hosts section) and print the exact cmds to create the VM and deploy:
+```sh
+./add.sh <HOST_NAME>                          # interactive for cpu/mem/disk/vlan
+./add.sh <HOST_NAME> --cpu 1 --mem 1024 --disk 10G --vlan 30
+```
 
-Modify the `per-host/<HOST_NAME>/configuration.nix` accordingly.
+Defaults for cpu/mem/disk/vlan are taken from `monitoring-2`. If the VM already exists in TF it is left alone.
 
-Generate host-keys, and `age` key for the host.
+The manual equivalent, if needed:
 ```sh
 ./generate-keys.sh <HOST_NAME>
+
+cat ./host-keys/<HOST_NAME>/ssh_host_ed25519_key.pub | ssh-to-age
 ```
 
+Copy the `age` key to the `.sops.yaml` config.
+
+Copy `per-host/<EXISTING_HOST>` to `per-host/<HOST_NAME>`.
+
+Modify the `per-host/<HOST_NAME>/configuration.nix` accordingly.
+
 ## Secrets w/ `sops`
 
 Edit the `.sops.yaml` to include the host's age key.
@@ -171,3 +183,8 @@ sudo garage bucket create <BUCKET_NAME>
 sudo garage bucket website --allow <BUCKET_NAME>
 sudo garage key create site-uploader
 ```
+
+### monitoring-2
+
+For monitoring.
+
diff --git a/nix/flake.nix b/nix/flake.nix
index 50d9f5f..eab43a1 100644
--- a/nix/flake.nix
+++ b/nix/flake.nix
@@ -71,6 +71,15 @@
           sops-nix.nixosModules.sops
         ];
       };
+
+      "monitoring-2" = nixpkgs.lib.nixosSystem {
+        system = "x86_64-linux";
+        modules = [
+          ./per-host/base/configuration.nix
+          ./per-host/monitoring-2/configuration.nix
+          sops-nix.nixosModules.sops
+        ];
+      };
     };
   };
 }
diff --git a/nix/per-host/monitoring-2/configuration.nix b/nix/per-host/monitoring-2/configuration.nix
new file mode 100644
index 0000000..ec210e1
--- /dev/null
+++ b/nix/per-host/monitoring-2/configuration.nix
@@ -0,0 +1,128 @@
+{ config, pkgs, modulesPath, lib, system, ... }:
+
+{
+  config = {
+    sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
+    sops.defaultSopsFile = ../../secrets/monitoring-2.yaml;
+
+    networking.hostName = "monitoring-2";
+    networking.firewall.allowedTCPPorts = [ 9090 ];
+
+    services.prometheus = {
+      enable = true;
+      port = 9090;
+
+      globalConfig = {
+        scrape_interval = "1m";
+        evaluation_interval = "1m";
+        # scrape_timeout is set to the global default (10s).
+        external_labels = {
+          monitor = "example";
+        };
+      };
+
+      # rule_files: none
+
+      scrapeConfigs = [
+        # The job name is added as a label `job=<job_name>` to any timeseries
+        # scraped from this config.
+        {
+          job_name = "prometheus";
+          # metrics_path defaults to '/metrics', scheme defaults to 'http'.
+          static_configs = [
+            { targets = [ "localhost:9090" ]; }
+          ];
+        }
+
+        {
+          job_name = "node";
+          # If prometheus-node-exporter is installed, grab stats about the
+          # local machine by default.
+          static_configs = [
+            { targets = [ "localhost:9100" "art-jr:9100" "vmbr0.bacala:9100" "vmbr0.art-sr:9100" "vmbr0.hesh:9100" ]; }
+          ];
+        }
+
+        {
+          job_name = "caddy";
+          static_configs = [
+            { targets = [ "rproxy-2:2019" ]; }
+          ];
+        }
+
+        {
+          job_name = "blackbox";
+          metrics_path = "/probe";
+          # the module to probe with comes from each target group's `module` label
+          static_configs = [
+            {
+              labels = { module = "http_2xx"; };
+              targets = [
+                "http://git:80" # git web
+                "http://caldav:5232" # radicale
+                "http://photos:2283" # immich
+                "http://med-dl:8686" # lidarr
+                "http://med-dl:7878/system/status" # radarr
+                "http://med-dl:8989/system/status" # sonarr
+                "http://vault:8989" # vaultwarden
+              ];
+            }
+            {
+              # auth-walled services: a 401/403 still means "up"
+              labels = { module = "http_up"; };
+              targets = [
+                "http://s3:3900/health" # garage
+                "http://torrents:9091" # transmission
+                "http://nas.bacala:8384" # syncthing
+              ];
+            }
+          ];
+          relabel_configs = [
+            {
+              source_labels = [ "__address__" ];
+              target_label = "__param_target";
+            }
+            {
+              source_labels = [ "__param_target" ];
+              target_label = "instance";
+            }
+            {
+              source_labels = [ "module" ];
+              target_label = "__param_module";
+            }
+            {
+              target_label = "__address__";
+              replacement = "127.0.0.1:9115"; # the blackbox exporter's real hostname:port
+            }
+          ];
+        }
+
+        {
+          job_name = "zfs_exporter";
+          static_configs = [
+            { targets = [ "art-jr:9134" "vmbr0.bacala:9134" ]; }
+          ];
+        }
+      ];
+
+      # scraped by the `node` job above
+      exporters.node.enable = true;
+
+      # scraped by the `blackbox` job above
+      exporters.blackbox = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        configFile = (pkgs.formats.yaml { }).generate "blackbox-exporter.yaml" {
+          modules = {
+            http_2xx.prober = "http";
+            http_up = {
+              prober = "http";
+              # a 401/403 response still proves the service is up
+              http.valid_status_codes = [ 401 403 ];
+            };
+          };
+        };
+      };
+    };
+  };
+}
diff --git a/nix/secrets/monitoring-2.yaml b/nix/secrets/monitoring-2.yaml
new file mode 100644
index 0000000..427f051
--- /dev/null
+++ b/nix/secrets/monitoring-2.yaml
@@ -0,0 +1,34 @@
+foo: ENC[AES256_GCM,data:afKy,iv:ydx5Tew0dNzJC0xDDkO2sfxIv7PmdS1lXFJmtSAuxsg=,tag:u1jqFOsa97YI2cLHeWl9OQ==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUQmU1aGJmVnhxbFBwZlgx
+            ZHBvamVyZHhCTVM0WEJsaVk3MFZqS0VseDF3Ck13S3J5VDJZWkYrOFFrRTVTekhp
+            aGhPQ2JoYmRpMndBMVNyOEVPcUtUT1kKLS0tIDJjQXhZVnlKTGY0RkRweGx5NDNm
+            eEU1WVRCbTVKL2h2M1VmV2pWWjdpZGMKE29XuIVVnDre65bYrzgHKNnHL+IPKNP1
+            PObQnh2fJgR+S7e+GkXlCu5fbKeFEEnfRJjE+lrd3Y1KmGU3+sVPoA==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age12teny70e5cm3992z7atx9g9606d4a60dsl8nskzkpp234uelnctq82mqaa
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBY3dWU3pRY3FESmJVTHNm
+            UHhQaW9mbnpaY3dteGJSVkFBR2ZqYXZBUm5JCmlyOG9vS2hsYlhjS2lkWFJtaysy
+            UWlQTWt2K2Q5dVhTVDlYOFg3MmlsYmsKLS0tIDhvdVZSNWFyQzB1clZ4eVJ2cnpv
+            eE1YSU5DcWRqTnZKc0U5d2RFNjBvNXcKWmWZu8ysMwrAzBlF9LHRjh9tsa7/ZMMu
+            DHl6+td7swegPYoA83g13VxbysMLQiVPdebaPGMolV/rObxASXX+yA==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age183q00vnssxx9ksdktsave78js2yggtcaahkf3qqh9j3yj88w54xq722xhy
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAza2ltam1HbmRJQVhSbVEz
+            bjd6ZXNVOXJ3RVJLRW1KbnE4WTR0OThxbVFjCkdMS2VVcU5QT09iV2RWZ2szVlBU
+            SFBCcnFOZ3ZKV3MzT25IZnBDVzFiNjgKLS0tIEEzUU1ubmc5R1JhWm5xYUZvcDVY
+            L0paNGJZbzhTcTF1QVpUTTRmUlpYbTgKv8+jdI7cKSBczLoL89MNas4lUaAZOyut
+            AfU6x5HavxUvCmxbc+QRyZRXhPzij84OPdl8Lljvws3d81elfNxBGw==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age177u6hs3funxd8eqd4h0krynrtn2et74m6dk2pk0q7rezw3w23dpsjtytte
+    lastmodified: "2026-09-15T16:39:28Z"
+    mac: ENC[AES256_GCM,data:89rYPn4rVXpvefiGG26H75TxD3WCKV4RX+q2oeh0DzsVAJpdJU+sYnCXWT7HYKYHqIQ2Nv2uhfnBDDdBnN4jrSzvbAQKavV41WFcyHC/xyigCZIO19EyVxwxaKeeD0CANENE8V1uf0KPvBgXdiyexXlvUzi1RL57xivc+QXJyeA=,iv:Gc0Wewn+YQm9E6ceKDifG4CQNc4HHuZ7/0/BZUuFyY8=,tag:BmFY5sLY5dZhUhEYWtmTGg==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.3
diff --git a/terraform/main.tf b/terraform/main.tf
index 5b83344..9d5f5f4 100755
--- a/terraform/main.tf
+++ b/terraform/main.tf
@@ -401,3 +401,38 @@ resource "proxmox_vm_qemu" "s3" {
     mtu    = 1420
   }
 }
+
+resource "proxmox_vm_qemu" "monitoring_2" {
+  target_node        = "hesh"
+  name               = "monitoring-2"
+  memory             = 1024
+  start_at_node_boot = true
+  boot               = "order=virtio0;net0"
+  clone_id           = 104
+  full_clone         = true
+  scsihw             = "virtio-scsi-single"
+
+  startup_shutdown {
+    order = 4
+  }
+
+  cpu {
+    cores = 1
+  }
+
+  disk {
+    type    = "disk"
+    slot    = "virtio0"
+    size    = "10G"
+    storage = "local-lvm"
+    format  = "raw"
+  }
+
+  network {
+    id     = 0
+    model  = "virtio"
+    bridge = "vmbr1"
+    tag    = 30
+    mtu    = 1420
+  }
+}