diff options
Diffstat (limited to 'nix')
| -rw-r--r-- | nix/.sops.yaml | 8 | ||||
| -rw-r--r-- | nix/README.md | 23 | ||||
| -rw-r--r-- | nix/flake.nix | 9 | ||||
| -rw-r--r-- | nix/per-host/monitoring-2/configuration.nix | 128 | ||||
| -rw-r--r-- | nix/secrets/monitoring-2.yaml | 34 |
5 files changed, 199 insertions, 3 deletions
diff --git a/nix/.sops.yaml b/nix/.sops.yaml index b657a25..cf20e88 100644 --- a/nix/.sops.yaml +++ b/nix/.sops.yaml @@ -7,6 +7,7 @@ keys: - &server_photos age1zdy4saxtcs6y90mrjlmz6hnknk67w0cd3n79g2y6fvtzzfsq6f6shwamzj - &server_caldav age1mhugjsg0pz45p56p5vc6hz05zrnk630j6p5xu4n2nsskvtxg8cqsndqr90 - &server_s3 age1nlkl8h7x33ssq3m5ka9syxurxz37y2duzhkvx3mmv23fxhetkuksqap73n + - &server_monitoring_2 age177u6hs3funxd8eqd4h0krynrtn2et74m6dk2pk0q7rezw3w23dpsjtytte creation_rules: - path_regex: 'secrets/torrents.*\.yaml$' @@ -50,3 +51,10 @@ creation_rules: - *admin_me - *admin_me_2 - *server_s3 + + - path_regex: 'secrets/monitoring.*\.yaml$' + key_groups: + - age: + - *admin_me + - *admin_me_2 + - *server_monitoring_2 diff --git a/nix/README.md b/nix/README.md index ab3a339..95cd4ac 100644 --- a/nix/README.md +++ b/nix/README.md @@ -38,15 +38,27 @@ tofu apply tfplan ## Add Host -Copy `per-host/<EXISTING_HOST>` to `per-host/<HOST_NAME>`. +Scaffold a new host (host keys, `.sops.yaml`, `flake.nix`, `per-host/`, secrets, TF VM if absent, `~/.ssh/config`, Hosts section) and print the exact cmds to create the VM and deploy: +```sh +./add.sh <HOST_NAME> # interactive for cpu/mem/disk/vlan +./add.sh <HOST_NAME> --cpu 1 --mem 1024 --disk 10G --vlan 30 +``` -Modify the `per-host/<HOST_NAME>/configuration.nix` accordingly. +Defaults for cpu/mem/disk/vlan are taken from `monitoring-2`. If the VM already exists in TF it is left alone. -Generate host-keys, and `age` key for the host. +The manual equivalent, if needed: ```sh ./generate-keys.sh <HOST_NAME> + +cat ./host-keys/<HOST_NAME>/ssh_host_ed25519_key.pub | ssh-to-age ``` +Copy the `age` key to the `.sops.yaml` config. + +Copy `per-host/<EXISTING_HOST>` to `per-host/<HOST_NAME>`. + +Modify the `per-host/<HOST_NAME>/configuration.nix` accordingly. + ## Secrets w/ `sops` Edit the `.sops.yaml` to include the host's age key. @@ -171,3 +183,8 @@ sudo garage bucket create <BUCKET_NAME> sudo garage bucket website --allow <BUCKET_NAME> sudo garage key create site-uploader ``` + +### monitoring-2 + +For monitoring. + diff --git a/nix/flake.nix b/nix/flake.nix index 50d9f5f..eab43a1 100644 --- a/nix/flake.nix +++ b/nix/flake.nix @@ -71,6 +71,15 @@ sops-nix.nixosModules.sops ]; }; + + "monitoring-2" = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [ + ./per-host/base/configuration.nix + ./per-host/monitoring-2/configuration.nix + sops-nix.nixosModules.sops + ]; + }; }; }; } diff --git a/nix/per-host/monitoring-2/configuration.nix b/nix/per-host/monitoring-2/configuration.nix new file mode 100644 index 0000000..ec210e1 --- /dev/null +++ b/nix/per-host/monitoring-2/configuration.nix @@ -0,0 +1,128 @@ +{ config, pkgs, modulesPath, lib, system, ... }: + +{ + config = { + sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ]; + sops.defaultSopsFile = ../../secrets/monitoring-2.yaml; + + networking.hostName = "monitoring-2"; + networking.firewall.allowedTCPPorts = [ 9090 ]; + + services.prometheus = { + enable = true; + port = 9090; + + globalConfig = { + scrape_interval = "1m"; + evaluation_interval = "1m"; + # scrape_timeout is set to the global default (10s). + external_labels = { + monitor = "example"; + }; + }; + + # rule_files: none + + scrapeConfigs = [ + # The job name is added as a label `job=<job_name>` to any timeseries + # scraped from this config. + { + job_name = "prometheus"; + # metrics_path defaults to '/metrics', scheme defaults to 'http'. + static_configs = [ + { targets = [ "localhost:9090" ]; } + ]; + } + + { + job_name = "node"; + # If prometheus-node-exporter is installed, grab stats about the + # local machine by default. + static_configs = [ + { targets = [ "localhost:9100" "art-jr:9100" "vmbr0.bacala:9100" "vmbr0.art-sr:9100" "vmbr0.hesh:9100" ]; } + ]; + } + + { + job_name = "caddy"; + static_configs = [ + { targets = [ "rproxy-2:2019" ]; } + ]; + } + + { + job_name = "blackbox"; + metrics_path = "/probe"; + # the module to probe with comes from each target group's `module` label + static_configs = [ + { + labels = { module = "http_2xx"; }; + targets = [ + "http://git:80" # git web + "http://caldav:5232" # radicale + "http://photos:2283" # immich + "http://med-dl:8686" # lidarr + "http://med-dl:7878/system/status" # radarr + "http://med-dl:8989/system/status" # sonarr + "http://vault:8989" # vaultwarden + ]; + } + { + # auth-walled services: a 401/403 still means "up" + labels = { module = "http_up"; }; + targets = [ + "http://s3:3900/health" # garage + "http://torrents:9091" # transmission + "http://nas.bacala:8384" # syncthing + ]; + } + ]; + relabel_configs = [ + { + source_labels = [ "__address__" ]; + target_label = "__param_target"; + } + { + source_labels = [ "__param_target" ]; + target_label = "instance"; + } + { + source_labels = [ "module" ]; + target_label = "__param_module"; + } + { + target_label = "__address__"; + replacement = "127.0.0.1:9115"; # the blackbox exporter's real hostname:port + } + ]; + } + + { + job_name = "zfs_exporter"; + static_configs = [ + { targets = [ "art-jr:9134" "vmbr0.bacala:9134" ]; } + ]; + } + ]; + + # scraped by the `node` job above + exporters.node.enable = true; + + # scraped by the `blackbox` job above + exporters.blackbox = { + enable = true; + listenAddress = "127.0.0.1"; + configFile = (pkgs.formats.yaml { }).generate "blackbox-exporter.yaml" { + modules = { + http_2xx.prober = "http"; + http_up = { + prober = "http"; + # a 401/403 response still proves the service is up + http.valid_status_codes = [ 401 403 ]; + }; + }; + }; + }; + }; + }; +} diff --git a/nix/secrets/monitoring-2.yaml b/nix/secrets/monitoring-2.yaml new file mode 100644 index 0000000..427f051 --- /dev/null +++ b/nix/secrets/monitoring-2.yaml @@ -0,0 +1,34 @@ +foo: ENC[AES256_GCM,data:afKy,iv:ydx5Tew0dNzJC0xDDkO2sfxIv7PmdS1lXFJmtSAuxsg=,tag:u1jqFOsa97YI2cLHeWl9OQ==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUQmU1aGJmVnhxbFBwZlgx + ZHBvamVyZHhCTVM0WEJsaVk3MFZqS0VseDF3Ck13S3J5VDJZWkYrOFFrRTVTekhp + aGhPQ2JoYmRpMndBMVNyOEVPcUtUT1kKLS0tIDJjQXhZVnlKTGY0RkRweGx5NDNm + eEU1WVRCbTVKL2h2M1VmV2pWWjdpZGMKE29XuIVVnDre65bYrzgHKNnHL+IPKNP1 + PObQnh2fJgR+S7e+GkXlCu5fbKeFEEnfRJjE+lrd3Y1KmGU3+sVPoA== + -----END AGE ENCRYPTED FILE----- + recipient: age12teny70e5cm3992z7atx9g9606d4a60dsl8nskzkpp234uelnctq82mqaa + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBY3dWU3pRY3FESmJVTHNm + UHhQaW9mbnpaY3dteGJSVkFBR2ZqYXZBUm5JCmlyOG9vS2hsYlhjS2lkWFJtaysy + UWlQTWt2K2Q5dVhTVDlYOFg3MmlsYmsKLS0tIDhvdVZSNWFyQzB1clZ4eVJ2cnpv + eE1YSU5DcWRqTnZKc0U5d2RFNjBvNXcKWmWZu8ysMwrAzBlF9LHRjh9tsa7/ZMMu + DHl6+td7swegPYoA83g13VxbysMLQiVPdebaPGMolV/rObxASXX+yA== + -----END AGE ENCRYPTED FILE----- + recipient: age183q00vnssxx9ksdktsave78js2yggtcaahkf3qqh9j3yj88w54xq722xhy + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAza2ltam1HbmRJQVhSbVEz + bjd6ZXNVOXJ3RVJLRW1KbnE4WTR0OThxbVFjCkdMS2VVcU5QT09iV2RWZ2szVlBU + SFBCcnFOZ3ZKV3MzT25IZnBDVzFiNjgKLS0tIEEzUU1ubmc5R1JhWm5xYUZvcDVY + L0paNGJZbzhTcTF1QVpUTTRmUlpYbTgKv8+jdI7cKSBczLoL89MNas4lUaAZOyut + AfU6x5HavxUvCmxbc+QRyZRXhPzij84OPdl8Lljvws3d81elfNxBGw== + -----END AGE ENCRYPTED FILE----- + recipient: age177u6hs3funxd8eqd4h0krynrtn2et74m6dk2pk0q7rezw3w23dpsjtytte + lastmodified: "2026-09-15T16:39:28Z" + mac: ENC[AES256_GCM,data:89rYPn4rVXpvefiGG26H75TxD3WCKV4RX+q2oeh0DzsVAJpdJU+sYnCXWT7HYKYHqIQ2Nv2uhfnBDDdBnN4jrSzvbAQKavV41WFcyHC/xyigCZIO19EyVxwxaKeeD0CANENE8V1uf0KPvBgXdiyexXlvUzi1RL57xivc+QXJyeA=,iv:Gc0Wewn+YQm9E6ceKDifG4CQNc4HHuZ7/0/BZUuFyY8=,tag:BmFY5sLY5dZhUhEYWtmTGg==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.3 |