basic prometheus alertmanager support

hosts/hopper/profiles/lab/caddy.nix

@@ -59,6 +59,7 @@ in {
       dash = mkPrivateEntry "dash" "${bridge}:${toString config.services.homepage-dashboard.listenPort}";
       absPriv = mkPrivateEntry "abs" "${bridge}:${toString config.services.audiobookshelf.port}";
       glances = mkPrivateEntry "glances" "${bridge}:${toString config.services.glances.port}";
+      alertmanager = mkPrivateEntry "alerts" "${bridge}:${toString config.services.prometheus.alertmanager.port}";
 
       base = {
         hostName = "${domain}:80";

hosts/hopper/profiles/lab/homepage.nix

@@ -67,6 +67,12 @@ in {
               };
             };
           }
+          {
+            "alertmanager" = {
+              href = "https://alerts.hopper.priv.${domain}";
+              icon = "prometheus";
+            };
+          }
           {
             "glances" = {
               href = "https://glances.hopper.priv.${domain}";

hosts/hopper/profiles/lab/prometheus.nix

@@ -3,10 +3,65 @@
   config,
   ...
 }: {
+  users.users."alertmanager" = {
+    group = "alertmanager";
+    isSystemUser = true;
+  };
+  users.groups."alertmanager" = {};
+
+  systemd.services.alertmanager.serviceConfig.User = "alertmanager";
+  systemd.services.alertmanager.serviceConfig.Group = "alertmanager";
+  systemd.services.alertmanager.serviceConfig.DynamicUser = lib.mkForce "false";
+
   services.prometheus = {
     enable = true;
     port = 9001;
     extraFlags = ["--storage.tsdb.retention.time=30d"];
+    alertmanager = {
+      enable = true;
+      configuration = {
+        route = {
+          group_by = ["alertname"];
+          receiver = "discord";
+        };
+        receivers = lib.singleton {
+          name = "discord";
+          discord_configs = lib.singleton {
+            webhook_url_file = config.sops.secrets.discord-webhook.path;
+          };
+        };
+      };
+    };
+    alertmanagers = lib.singleton {
+      scheme = "http";
+      static_configs = lib.singleton {
+        targets = ["localhost:${toString config.services.prometheus.alertmanager.port}"];
+      };
+    };
+    rules = let
+      megabyte = builtins.mul (1024 * 1024);
+    in [
+      ''
+        groups:
+          - name: disk
+            rules:
+            - alert: DiskSpaceLow
+              expr: |
+                node_filesystem_avail_bytes{fstype!~"(tmpfs|ramfs|fuse.*)"} / node_filesystem_size_bytes < 0.05
+                OR
+                node_filesystem_avail_bytes{fstype!~"(tmpfs|ramfs|fuse.*)"} < ${toString (megabyte 100)}
+              for: 5m
+              annotations:
+                summary: "low disk space on {{ $labels.alias }} (mountpoint: {{ $labels.mountpoint }})"
+          - name: systemd
+            rules:
+            - alert: FailedUnits
+              expr: node_systemd_unit_state{state="failed"} == 1
+              for: 5m
+              annotations:
+                summary: "service {{ $labels.name }} failed on {{ $labels.alias }}"
+      ''
+    ];
     scrapeConfigs = [
       {
         job_name = "node";
@@ -58,15 +113,14 @@
         };
       }
     ];
-  };
+    exporters = {
-
-  services.prometheus.exporters = {
       node = {
         enable = true;
         enabledCollectors = ["systemd"];
       };
       systemd.enable = true;
     };
+  };
 
   environment.persistence."/persist".directories = ["/var/lib/prometheus2"];
   services.restic.backups.hopper.paths = ["/var/lib/prometheus2"];

hosts/hopper/profiles/lab/vpn-namespace.nix

@@ -6,6 +6,7 @@
   networking.firewall = let
     allowTcpFromVPNToDefaultPorts = [
       config.services.prometheus.port
+      config.services.prometheus.alertmanager.port
       config.services.vaultwarden.config.ROCKET_PORT
       config.services.homepage-dashboard.listenPort
       config.services.audiobookshelf.port

secrets/hopper/default.nix

@@ -5,6 +5,13 @@
       format = "binary";
       sopsFile = ./wireguard;
     };
+    discord-webhook = {
+      format = "binary";
+      owner = "alertmanager";
+      group = "alertmanager";
+      sopsFile = ./discord-webhook;
+      restartUnits = ["alertmanager.service"];
+    };
     slskd = {
       format = "binary";
       sopsFile = ./slskd;

secrets/hopper/discord-webhook

@@ -0,0 +1,19 @@
+{
+	"data": "ENC[AES256_GCM,data:rnxEDYYXsMi4uI426ffiGuyrcONgJ2lpolu5mzKOlu7/FrT7BbKYoJIJ2T2/SuPY1W2PtMFWl+lCZrh5fyZ8Z0mK4+5Gi6DFzZYOFWe1PJAYTqaYcRlmWSFA2ga5JbwXX0UP4iY/wd0PaB3r+XA88Cp2OMchlSHgTqA=,iv:2a0iqOHwoXZSftM7hd9TgL/nU5J0cGvaH62zkS4Iqt8=,tag:MNgm76nPh5GkS7SHD4B/Cw==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"recipient": "age17pdqkpfh6kc6wm7gxzdnwf6vphlwddv9yfpdu3j76e24y3amd9tq3avfc8",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUWDhxZFRKaFV3Qm10eEhV\nbzFYbDB2cHJxb2NUVCtnK1VYcUt5V1NXV0JrCnN3dy9qVVVsRmtoc2lyWW1GZFh4\nNXFjbzkxS2lRNkNtQnE1eHg3TUJYL3cKLS0tIEpWUzJYbWNnbjlabkhIanNjdjU2\nRHVaa3QyL2Y1eW1kUjVZTlFjOEZ4bm8KUECVAFF6th9EQRnCrVTHmIj6QAsQbpTO\nND0XUrqnrnPot8GARyTGxBkQJUymbra67TBtMwnmX0LYpg+FHvvL9A==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1e9nhfwfcg9krc03re4fwh0wu0cwf6jq4js5vfn26hcdqc2apgdes98fea7",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIY1lCRERGbU9qOFBteGM1\nVDh0RUlvMWNLZ0Y0UHFDTFRtbXRzajdUMW5JClNFNmVRT1c0amtxRzV5am9nakgv\nVi84eEFXZEJlaURpQWZDQnV3OFRVOTQKLS0tIEdacVRDT09XYVY5VWlJN3RORFpu\nMTlKK0lIYmkwdlMwMFp5bENvUnNVcUkKJmWZ/eVlc7xZrE1bZBCV7AR/wTx9vOVv\n1ThEEFLFMtBJxE4bXMoYtY5SGWFC9AihS+ES1IvhFv8ZbRwvffe18g==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2025-06-20T13:30:10Z",
+		"mac": "ENC[AES256_GCM,data:YWvh/lXSRz91J0NF1COLAzFz9NVo09tul9MAqEY9uLp3/E1SJ2ekgYBJruoaR2nPj6rYVj+UD1+l7Ak6jQWcunLw7YhEXzklrGHAGmCm8WIaen9rL196TKXs7ADtzohovlBW12dLTFZK7Qo8mXuU5cBhN0DEsec6fZ/fCaDcyYc=,iv:e922Mv/UaWv10qsjDXbu58/sO2MbJPpiA132rPs3Azk=,tag:nNPtrUR1eUYkVF98ZDoy4Q==,type:str]",
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.10.2"
+	}
+}