monitoring config

1 files changed, 128 insertions, 0 deletions

diff --git a/nix/per-host/monitoring-2/configuration.nix b/nix/per-host/monitoring-2/configuration.nix
new file mode 100644
index 0000000..ec210e1
--- /dev/null
+++ b/nix/per-host/monitoring-2/configuration.nix
@@ -0,0 +1,128 @@
+{ config, pkgs, modulesPath, lib, system, ... }:
+
+{
+  config = {
+    sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
+    sops.defaultSopsFile = ../../secrets/monitoring-2.yaml;
+
+    networking.hostName = "monitoring-2";
+    networking.firewall.allowedTCPPorts = [ 9090 ];
+
+    services.prometheus = {
+      enable = true;
+      port = 9090;
+
+      globalConfig = {
+        scrape_interval = "1m";
+        evaluation_interval = "1m";
+        # scrape_timeout is set to the global default (10s).
+        external_labels = {
+          monitor = "example";
+        };
+      };
+
+      # rule_files: none
+
+      scrapeConfigs = [
+        # The job name is added as a label `job=<job_name>` to any timeseries
+        # scraped from this config.
+        {
+          job_name = "prometheus";
+          # metrics_path defaults to '/metrics', scheme defaults to 'http'.
+          static_configs = [
+            { targets = [ "localhost:9090" ]; }
+          ];
+        }
+
+        {
+          job_name = "node";
+          # If prometheus-node-exporter is installed, grab stats about the
+          # local machine by default.
+          static_configs = [
+            { targets = [ "localhost:9100" "art-jr:9100" "vmbr0.bacala:9100" "vmbr0.art-sr:9100" "vmbr0.hesh:9100" ]; }
+          ];
+        }
+
+        {
+          job_name = "caddy";
+          static_configs = [
+            { targets = [ "rproxy-2:2019" ]; }
+          ];
+        }
+
+        {
+          job_name = "blackbox";
+          metrics_path = "/probe";
+          # the module to probe with comes from each target group's `module` label
+          static_configs = [
+            {
+              labels = { module = "http_2xx"; };
+              targets = [
+                "http://git:80" # git web
+                "http://caldav:5232" # radicale
+                "http://photos:2283" # immich
+                "http://med-dl:8686" # lidarr
+                "http://med-dl:7878/system/status" # radarr
+                "http://med-dl:8989/system/status" # sonarr
+                "http://vault:8989" # vaultwarden
+              ];
+            }
+            {
+              # auth-walled services: a 401/403 still means "up"
+              labels = { module = "http_up"; };
+              targets = [
+                "http://s3:3900/health" # garage
+                "http://torrents:9091" # transmission
+                "http://nas.bacala:8384" # syncthing
+              ];
+            }
+          ];
+          relabel_configs = [
+            {
+              source_labels = [ "__address__" ];
+              target_label = "__param_target";
+            }
+            {
+              source_labels = [ "__param_target" ];
+              target_label = "instance";
+            }
+            {
+              source_labels = [ "module" ];
+              target_label = "__param_module";
+            }
+            {
+              target_label = "__address__";
+              replacement = "127.0.0.1:9115"; # the blackbox exporter's real hostname:port
+            }
+          ];
+        }
+
+        {
+          job_name = "zfs_exporter";
+          static_configs = [
+            { targets = [ "art-jr:9134" "vmbr0.bacala:9134" ]; }
+          ];
+        }
+      ];
+
+      # scraped by the `node` job above
+      exporters.node.enable = true;
+
+      # scraped by the `blackbox` job above
+      exporters.blackbox = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        configFile = (pkgs.formats.yaml { }).generate "blackbox-exporter.yaml" {
+          modules = {
+            http_2xx.prober = "http";
+            http_up = {
+              prober = "http";
+              # a 401/403 response still proves the service is up
+              http.valid_status_codes = [ 401 403 ];
+            };
+          };
+        };
+      };
+    };
+  };
+}