xun/nixos-config
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

basic prometheus alertmanager support

Browse source
This commit is contained in:
xunuwu 2025-06-20 13:20:57 +02:00
parent 5c0dd03f27
commit e2ffa8003f
Signed by: xun
SSH key fingerprint: SHA256:Uot/1WoAjWAeqLOHA5vYy4phhVydsH7jCPmBjaPZfgI
6 changed files with 95 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

1
hosts/hopper/profiles/lab/caddy.nix
View file

@ -59,6 +59,7 @@ in {
dash = mkPrivateEntry "dash" "${bridge}:${toString config.services.homepage-dashboard.listenPort}";
absPriv = mkPrivateEntry "abs" "${bridge}:${toString config.services.audiobookshelf.port}";
glances = mkPrivateEntry "glances" "${bridge}:${toString config.services.glances.port}";
alertmanager = mkPrivateEntry "alerts" "${bridge}:${toString config.services.prometheus.alertmanager.port}";
base = {
hostName = "${domain}:80";

6
hosts/hopper/profiles/lab/homepage.nix
View file

@ -67,6 +67,12 @@ in {
};
};
}
{
"alertmanager" = {
href = "https://alerts.hopper.priv.${domain}";
icon = "prometheus";
};
}
{
"glances" = {
href = "https://glances.hopper.priv.${domain}";

68
hosts/hopper/profiles/lab/prometheus.nix
View file

@ -3,10 +3,65 @@
config,
...
}: {
users.users."alertmanager" = {
group = "alertmanager";
isSystemUser = true;
};
users.groups."alertmanager" = {};
systemd.services.alertmanager.serviceConfig.User = "alertmanager";
systemd.services.alertmanager.serviceConfig.Group = "alertmanager";
systemd.services.alertmanager.serviceConfig.DynamicUser = lib.mkForce "false";
services.prometheus = {
enable = true;
port = 9001;
extraFlags = ["--storage.tsdb.retention.time=30d"];
alertmanager = {
enable = true;
configuration = {
route = {
group_by = ["alertname"];
receiver = "discord";
};
receivers = lib.singleton {
name = "discord";
discord_configs = lib.singleton {
webhook_url_file = config.sops.secrets.discord-webhook.path;
};
};
};
};
alertmanagers = lib.singleton {
scheme = "http";
static_configs = lib.singleton {
targets = ["localhost:${toString config.services.prometheus.alertmanager.port}"];
};
};
rules = let
megabyte = builtins.mul (1024 * 1024);
in [
''
groups:
- name: disk
rules:
- alert: DiskSpaceLow
expr: |
node_filesystem_avail_bytes{fstype!~"(tmpfs|ramfs|fuse.*)"} / node_filesystem_size_bytes < 0.05
OR
node_filesystem_avail_bytes{fstype!~"(tmpfs|ramfs|fuse.*)"} < ${toString (megabyte 100)}
for: 5m
annotations:
summary: "low disk space on {{ $labels.alias }} (mountpoint: {{ $labels.mountpoint }})"
- name: systemd
rules:
- alert: FailedUnits
expr: node_systemd_unit_state{state="failed"} == 1
for: 5m
annotations:
summary: "service {{ $labels.name }} failed on {{ $labels.alias }}"
''
];
scrapeConfigs = [
{
job_name = "node";
@ -58,14 +113,13 @@
};
}
];
};
services.prometheus.exporters = {
node = {
enable = true;
enabledCollectors = ["systemd"];
exporters = {
node = {
enable = true;
enabledCollectors = ["systemd"];
};
systemd.enable = true;
};
systemd.enable = true;
};
environment.persistence."/persist".directories = ["/var/lib/prometheus2"];

1
hosts/hopper/profiles/lab/vpn-namespace.nix
View file

@ -6,6 +6,7 @@
networking.firewall = let
allowTcpFromVPNToDefaultPorts = [
config.services.prometheus.port
config.services.prometheus.alertmanager.port
config.services.vaultwarden.config.ROCKET_PORT
config.services.homepage-dashboard.listenPort
config.services.audiobookshelf.port

7
secrets/hopper/default.nix
View file

@ -5,6 +5,13 @@
format = "binary";
sopsFile = ./wireguard;
};
discord-webhook = {
format = "binary";
owner = "alertmanager";
group = "alertmanager";
sopsFile = ./discord-webhook;
restartUnits = ["alertmanager.service"];
};
slskd = {
format = "binary";
sopsFile = ./slskd;

19
secrets/hopper/discord-webhook Normal file
View file

@ -0,0 +1,19 @@
{
"data": "ENC[AES256_GCM,data:rnxEDYYXsMi4uI426ffiGuyrcONgJ2lpolu5mzKOlu7/FrT7BbKYoJIJ2T2/SuPY1W2PtMFWl+lCZrh5fyZ8Z0mK4+5Gi6DFzZYOFWe1PJAYTqaYcRlmWSFA2ga5JbwXX0UP4iY/wd0PaB3r+XA88Cp2OMchlSHgTqA=,iv:2a0iqOHwoXZSftM7hd9TgL/nU5J0cGvaH62zkS4Iqt8=,tag:MNgm76nPh5GkS7SHD4B/Cw==,type:str]",
"sops": {
"age": [
{
"recipient": "age17pdqkpfh6kc6wm7gxzdnwf6vphlwddv9yfpdu3j76e24y3amd9tq3avfc8",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUWDhxZFRKaFV3Qm10eEhV\nbzFYbDB2cHJxb2NUVCtnK1VYcUt5V1NXV0JrCnN3dy9qVVVsRmtoc2lyWW1GZFh4\nNXFjbzkxS2lRNkNtQnE1eHg3TUJYL3cKLS0tIEpWUzJYbWNnbjlabkhIanNjdjU2\nRHVaa3QyL2Y1eW1kUjVZTlFjOEZ4bm8KUECVAFF6th9EQRnCrVTHmIj6QAsQbpTO\nND0XUrqnrnPot8GARyTGxBkQJUymbra67TBtMwnmX0LYpg+FHvvL9A==\n-----END AGE ENCRYPTED FILE-----\n"
},
{
"recipient": "age1e9nhfwfcg9krc03re4fwh0wu0cwf6jq4js5vfn26hcdqc2apgdes98fea7",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIY1lCRERGbU9qOFBteGM1\nVDh0RUlvMWNLZ0Y0UHFDTFRtbXRzajdUMW5JClNFNmVRT1c0amtxRzV5am9nakgv\nVi84eEFXZEJlaURpQWZDQnV3OFRVOTQKLS0tIEdacVRDT09XYVY5VWlJN3RORFpu\nMTlKK0lIYmkwdlMwMFp5bENvUnNVcUkKJmWZ/eVlc7xZrE1bZBCV7AR/wTx9vOVv\n1ThEEFLFMtBJxE4bXMoYtY5SGWFC9AihS+ES1IvhFv8ZbRwvffe18g==\n-----END AGE ENCRYPTED FILE-----\n"
}
],
"lastmodified": "2025-06-20T13:30:10Z",
"mac": "ENC[AES256_GCM,data:YWvh/lXSRz91J0NF1COLAzFz9NVo09tul9MAqEY9uLp3/E1SJ2ekgYBJruoaR2nPj6rYVj+UD1+l7Ak6jQWcunLw7YhEXzklrGHAGmCm8WIaen9rL196TKXs7ADtzohovlBW12dLTFZK7Qo8mXuU5cBhN0DEsec6fZ/fCaDcyYc=,iv:e922Mv/UaWv10qsjDXbu58/sO2MbJPpiA132rPs3Azk=,tag:nNPtrUR1eUYkVF98ZDoy4Q==,type:str]",
"unencrypted_suffix": "_unencrypted",
"version": "3.10.2"
}
}