{ config, pkgs, modulesPath, lib, system, ... }:

{
  config = {
    sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
    sops.defaultSopsFile = ../../secrets/monitoring-2.yaml;

    networking.hostName = "monitoring-2";
    networking.firewall.allowedTCPPorts = [ 9090 ];

    services.prometheus = {
      enable = true;
      port = 9090;

      globalConfig = {
        scrape_interval = "1m";
        evaluation_interval = "1m";
        # scrape_timeout is set to the global default (10s).
        external_labels = {
          monitor = "example";
        };
      };

      # rule_files: none

      scrapeConfigs = [
        # The job name is added as a label `job=<job_name>` to any timeseries
        # scraped from this config.
        {
          job_name = "prometheus";
          # metrics_path defaults to '/metrics', scheme defaults to 'http'.
          static_configs = [
            { targets = [ "localhost:9090" ]; }
          ];
        }

        {
          job_name = "node";
          # If prometheus-node-exporter is installed, grab stats about the
          # local machine by default.
          static_configs = [
            { targets = [ "localhost:9100" "art-jr:9100" "vmbr0.bacala:9100" "vmbr0.art-sr:9100" "vmbr0.hesh:9100" ]; }
          ];
        }

        {
          job_name = "caddy";
          static_configs = [
            { targets = [ "rproxy-2:2019" ]; }
          ];
        }

        {
          job_name = "blackbox";
          metrics_path = "/probe";
          # the module to probe with comes from each target group's `module` label
          static_configs = [
            {
              labels = { module = "http_2xx"; };
              targets = [
                "http://git:80" # git web
                "http://caldav:5232" # radicale
                "http://photos:2283" # immich
                "http://med-dl:8686" # lidarr
                "http://med-dl:7878/system/status" # radarr
                "http://med-dl:8989/system/status" # sonarr
                "http://vault:8989" # vaultwarden
              ];
            }
            {
              # auth-walled services: a 401/403 still means "up"
              labels = { module = "http_up"; };
              targets = [
                "http://s3:3900/health" # garage
                "http://torrents:9091" # transmission
                "http://nas.bacala:8384" # syncthing
              ];
            }
          ];
          relabel_configs = [
            {
              source_labels = [ "__address__" ];
              target_label = "__param_target";
            }
            {
              source_labels = [ "__param_target" ];
              target_label = "instance";
            }
            {
              source_labels = [ "module" ];
              target_label = "__param_module";
            }
            {
              target_label = "__address__";
              replacement = "127.0.0.1:9115"; # the blackbox exporter's real hostname:port
            }
          ];
        }

        {
          job_name = "zfs_exporter";
          static_configs = [
            { targets = [ "art-jr:9134" "vmbr0.bacala:9134" ]; }
          ];
        }
      ];

      # scraped by the `node` job above
      exporters.node.enable = true;

      # scraped by the `blackbox` job above
      exporters.blackbox = {
        enable = true;
        listenAddress = "127.0.0.1";
        configFile = (pkgs.formats.yaml { }).generate "blackbox-exporter.yaml" {
          modules = {
            http_2xx.prober = "http";
            http_up = {
              prober = "http";
              # a 401/403 response still proves the service is up
              http.valid_status_codes = [ 401 403 ];
            };
          };
        };
      };
    };
  };
}