blob: ec210e103a9a7c5b5af3680fcecadcad1590c49d (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
{ config, pkgs, modulesPath, lib, system, ... }:
{
config = {
sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
sops.defaultSopsFile = ../../secrets/monitoring-2.yaml;
networking.hostName = "monitoring-2";
networking.firewall.allowedTCPPorts = [ 9090 ];
services.prometheus = {
enable = true;
port = 9090;
globalConfig = {
scrape_interval = "1m";
evaluation_interval = "1m";
# scrape_timeout is set to the global default (10s).
external_labels = {
monitor = "example";
};
};
# rule_files: none
scrapeConfigs = [
# The job name is added as a label `job=<job_name>` to any timeseries
# scraped from this config.
{
job_name = "prometheus";
# metrics_path defaults to '/metrics', scheme defaults to 'http'.
static_configs = [
{ targets = [ "localhost:9090" ]; }
];
}
{
job_name = "node";
# If prometheus-node-exporter is installed, grab stats about the
# local machine by default.
static_configs = [
{ targets = [ "localhost:9100" "art-jr:9100" "vmbr0.bacala:9100" "vmbr0.art-sr:9100" "vmbr0.hesh:9100" ]; }
];
}
{
job_name = "caddy";
static_configs = [
{ targets = [ "rproxy-2:2019" ]; }
];
}
{
job_name = "blackbox";
metrics_path = "/probe";
# the module to probe with comes from each target group's `module` label
static_configs = [
{
labels = { module = "http_2xx"; };
targets = [
"http://git:80" # git web
"http://caldav:5232" # radicale
"http://photos:2283" # immich
"http://med-dl:8686" # lidarr
"http://med-dl:7878/system/status" # radarr
"http://med-dl:8989/system/status" # sonarr
"http://vault:8989" # vaultwarden
];
}
{
# auth-walled services: a 401/403 still means "up"
labels = { module = "http_up"; };
targets = [
"http://s3:3900/health" # garage
"http://torrents:9091" # transmission
"http://nas.bacala:8384" # syncthing
];
}
];
relabel_configs = [
{
source_labels = [ "__address__" ];
target_label = "__param_target";
}
{
source_labels = [ "__param_target" ];
target_label = "instance";
}
{
source_labels = [ "module" ];
target_label = "__param_module";
}
{
target_label = "__address__";
replacement = "127.0.0.1:9115"; # the blackbox exporter's real hostname:port
}
];
}
{
job_name = "zfs_exporter";
static_configs = [
{ targets = [ "art-jr:9134" "vmbr0.bacala:9134" ]; }
];
}
];
# scraped by the `node` job above
exporters.node.enable = true;
# scraped by the `blackbox` job above
exporters.blackbox = {
enable = true;
listenAddress = "127.0.0.1";
configFile = (pkgs.formats.yaml { }).generate "blackbox-exporter.yaml" {
modules = {
http_2xx.prober = "http";
http_up = {
prober = "http";
# a 401/403 response still proves the service is up
http.valid_status_codes = [ 401 403 ];
};
};
};
};
};
};
}
|
