diff --git a/hosts/hopper/profiles/lab/caddy.nix b/hosts/hopper/profiles/lab/caddy.nix index f58fc12..d549397 100644 --- a/hosts/hopper/profiles/lab/caddy.nix +++ b/hosts/hopper/profiles/lab/caddy.nix @@ -59,6 +59,7 @@ in { dash = mkPrivateEntry "dash" "${bridge}:${toString config.services.homepage-dashboard.listenPort}"; absPriv = mkPrivateEntry "abs" "${bridge}:${toString config.services.audiobookshelf.port}"; glances = mkPrivateEntry "glances" "${bridge}:${toString config.services.glances.port}"; + alertmanager = mkPrivateEntry "alerts" "${bridge}:${toString config.services.prometheus.alertmanager.port}"; base = { hostName = "${domain}:80"; diff --git a/hosts/hopper/profiles/lab/homepage.nix b/hosts/hopper/profiles/lab/homepage.nix index 43085ac..e6201b7 100644 --- a/hosts/hopper/profiles/lab/homepage.nix +++ b/hosts/hopper/profiles/lab/homepage.nix @@ -67,6 +67,12 @@ in { }; }; } + { + "alertmanager" = { + href = "https://alerts.hopper.priv.${domain}"; + icon = "prometheus"; + }; + } { "glances" = { href = "https://glances.hopper.priv.${domain}"; diff --git a/hosts/hopper/profiles/lab/prometheus.nix b/hosts/hopper/profiles/lab/prometheus.nix index fdff915..315548a 100644 --- a/hosts/hopper/profiles/lab/prometheus.nix +++ b/hosts/hopper/profiles/lab/prometheus.nix @@ -3,10 +3,65 @@ config, ... }: { + users.users."alertmanager" = { + group = "alertmanager"; + isSystemUser = true; + }; + users.groups."alertmanager" = {}; + + systemd.services.alertmanager.serviceConfig.User = "alertmanager"; + systemd.services.alertmanager.serviceConfig.Group = "alertmanager"; + systemd.services.alertmanager.serviceConfig.DynamicUser = lib.mkForce "false"; + services.prometheus = { enable = true; port = 9001; extraFlags = ["--storage.tsdb.retention.time=30d"]; + alertmanager = { + enable = true; + configuration = { + route = { + group_by = ["alertname"]; + receiver = "discord"; + }; + receivers = lib.singleton { + name = "discord"; + discord_configs = lib.singleton { + webhook_url_file = config.sops.secrets.discord-webhook.path; + }; + }; + }; + }; + alertmanagers = lib.singleton { + scheme = "http"; + static_configs = lib.singleton { + targets = ["localhost:${toString config.services.prometheus.alertmanager.port}"]; + }; + }; + rules = let + megabyte = builtins.mul (1024 * 1024); + in [ + '' + groups: + - name: disk + rules: + - alert: DiskSpaceLow + expr: | + node_filesystem_avail_bytes{fstype!~"(tmpfs|ramfs|fuse.*)"} / node_filesystem_size_bytes < 0.05 + OR + node_filesystem_avail_bytes{fstype!~"(tmpfs|ramfs|fuse.*)"} < ${toString (megabyte 100)} + for: 5m + annotations: + summary: "low disk space on {{ $labels.alias }} (mountpoint: {{ $labels.mountpoint }})" + - name: systemd + rules: + - alert: FailedUnits + expr: node_systemd_unit_state{state="failed"} == 1 + for: 5m + annotations: + summary: "service {{ $labels.name }} failed on {{ $labels.alias }}" + '' + ]; scrapeConfigs = [ { job_name = "node"; @@ -58,14 +113,13 @@ }; } ]; - }; - - services.prometheus.exporters = { - node = { - enable = true; - enabledCollectors = ["systemd"]; + exporters = { + node = { + enable = true; + enabledCollectors = ["systemd"]; + }; + systemd.enable = true; }; - systemd.enable = true; }; environment.persistence."/persist".directories = ["/var/lib/prometheus2"]; diff --git a/hosts/hopper/profiles/lab/vpn-namespace.nix b/hosts/hopper/profiles/lab/vpn-namespace.nix index 6b8b5e0..84375fe 100644 --- a/hosts/hopper/profiles/lab/vpn-namespace.nix +++ b/hosts/hopper/profiles/lab/vpn-namespace.nix @@ -6,6 +6,7 @@ networking.firewall = let allowTcpFromVPNToDefaultPorts = [ config.services.prometheus.port + config.services.prometheus.alertmanager.port config.services.vaultwarden.config.ROCKET_PORT config.services.homepage-dashboard.listenPort config.services.audiobookshelf.port diff --git a/secrets/hopper/default.nix b/secrets/hopper/default.nix index 4b89556..efa968d 100644 --- a/secrets/hopper/default.nix +++ b/secrets/hopper/default.nix @@ -5,6 +5,13 @@ format = "binary"; sopsFile = ./wireguard; }; + discord-webhook = { + format = "binary"; + owner = "alertmanager"; + group = "alertmanager"; + sopsFile = ./discord-webhook; + restartUnits = ["alertmanager.service"]; + }; slskd = { format = "binary"; sopsFile = ./slskd; diff --git a/secrets/hopper/discord-webhook b/secrets/hopper/discord-webhook new file mode 100644 index 0000000..32fb90b --- /dev/null +++ b/secrets/hopper/discord-webhook @@ -0,0 +1,19 @@ +{ + "data": "ENC[AES256_GCM,data:rnxEDYYXsMi4uI426ffiGuyrcONgJ2lpolu5mzKOlu7/FrT7BbKYoJIJ2T2/SuPY1W2PtMFWl+lCZrh5fyZ8Z0mK4+5Gi6DFzZYOFWe1PJAYTqaYcRlmWSFA2ga5JbwXX0UP4iY/wd0PaB3r+XA88Cp2OMchlSHgTqA=,iv:2a0iqOHwoXZSftM7hd9TgL/nU5J0cGvaH62zkS4Iqt8=,tag:MNgm76nPh5GkS7SHD4B/Cw==,type:str]", + "sops": { + "age": [ + { + "recipient": "age17pdqkpfh6kc6wm7gxzdnwf6vphlwddv9yfpdu3j76e24y3amd9tq3avfc8", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUWDhxZFRKaFV3Qm10eEhV\nbzFYbDB2cHJxb2NUVCtnK1VYcUt5V1NXV0JrCnN3dy9qVVVsRmtoc2lyWW1GZFh4\nNXFjbzkxS2lRNkNtQnE1eHg3TUJYL3cKLS0tIEpWUzJYbWNnbjlabkhIanNjdjU2\nRHVaa3QyL2Y1eW1kUjVZTlFjOEZ4bm8KUECVAFF6th9EQRnCrVTHmIj6QAsQbpTO\nND0XUrqnrnPot8GARyTGxBkQJUymbra67TBtMwnmX0LYpg+FHvvL9A==\n-----END AGE ENCRYPTED FILE-----\n" + }, + { + "recipient": "age1e9nhfwfcg9krc03re4fwh0wu0cwf6jq4js5vfn26hcdqc2apgdes98fea7", + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIY1lCRERGbU9qOFBteGM1\nVDh0RUlvMWNLZ0Y0UHFDTFRtbXRzajdUMW5JClNFNmVRT1c0amtxRzV5am9nakgv\nVi84eEFXZEJlaURpQWZDQnV3OFRVOTQKLS0tIEdacVRDT09XYVY5VWlJN3RORFpu\nMTlKK0lIYmkwdlMwMFp5bENvUnNVcUkKJmWZ/eVlc7xZrE1bZBCV7AR/wTx9vOVv\n1ThEEFLFMtBJxE4bXMoYtY5SGWFC9AihS+ES1IvhFv8ZbRwvffe18g==\n-----END AGE ENCRYPTED FILE-----\n" + } + ], + "lastmodified": "2025-06-20T13:30:10Z", + "mac": "ENC[AES256_GCM,data:YWvh/lXSRz91J0NF1COLAzFz9NVo09tul9MAqEY9uLp3/E1SJ2ekgYBJruoaR2nPj6rYVj+UD1+l7Ak6jQWcunLw7YhEXzklrGHAGmCm8WIaen9rL196TKXs7ADtzohovlBW12dLTFZK7Qo8mXuU5cBhN0DEsec6fZ/fCaDcyYc=,iv:e922Mv/UaWv10qsjDXbu58/sO2MbJPpiA132rPs3Azk=,tag:nNPtrUR1eUYkVF98ZDoy4Q==,type:str]", + "unencrypted_suffix": "_unencrypted", + "version": "3.10.2" + } +}