basic prometheus alertmanager support

hosts/hopper/profiles/lab/caddy.nix

@@ -59,6 +59,7 @@ in {
       dash = mkPrivateEntry "dash" "${bridge}:${toString config.services.homepage-dashboard.listenPort}";
       absPriv = mkPrivateEntry "abs" "${bridge}:${toString config.services.audiobookshelf.port}";
       glances = mkPrivateEntry "glances" "${bridge}:${toString config.services.glances.port}";
+      alertmanager = mkPrivateEntry "alerts" "${bridge}:${toString config.services.prometheus.alertmanager.port}";
 
       base = {
         hostName = "${domain}:80";

hosts/hopper/profiles/lab/homepage.nix

@@ -67,6 +67,12 @@ in {
               };
             };
           }
+          {
+            "alertmanager" = {
+              href = "https://alerts.hopper.priv.${domain}";
+              icon = "prometheus";
+            };
+          }
           {
             "glances" = {
               href = "https://glances.hopper.priv.${domain}";

hosts/hopper/profiles/lab/prometheus.nix

@@ -3,10 +3,65 @@
   config,
   ...
 }: {
+  users.users."alertmanager" = {
+    group = "alertmanager";
+    isSystemUser = true;
+  };
+  users.groups."alertmanager" = {};
+
+  systemd.services.alertmanager.serviceConfig.User = "alertmanager";
+  systemd.services.alertmanager.serviceConfig.Group = "alertmanager";
+  systemd.services.alertmanager.serviceConfig.DynamicUser = lib.mkForce "false";
+
   services.prometheus = {
     enable = true;
     port = 9001;
     extraFlags = ["--storage.tsdb.retention.time=30d"];
+    alertmanager = {
+      enable = true;
+      configuration = {
+        route = {
+          group_by = ["alertname"];
+          receiver = "discord";
+        };
+        receivers = lib.singleton {
+          name = "discord";
+          discord_configs = lib.singleton {
+            webhook_url_file = config.sops.secrets.discord-webhook.path;
+          };
+        };
+      };
+    };
+    alertmanagers = lib.singleton {
+      scheme = "http";
+      static_configs = lib.singleton {
+        targets = ["localhost:${toString config.services.prometheus.alertmanager.port}"];
+      };
+    };
+    rules = let
+      megabyte = builtins.mul (1024 * 1024);
+    in [
+      ''
+        groups:
+          - name: disk
+            rules:
+            - alert: DiskSpaceLow
+              expr: |
+                node_filesystem_avail_bytes{fstype!~"(tmpfs|ramfs|fuse.*)"} / node_filesystem_size_bytes < 0.05
+                OR
+                node_filesystem_avail_bytes{fstype!~"(tmpfs|ramfs|fuse.*)"} < ${toString (megabyte 100)}
+              for: 5m
+              annotations:
+                summary: "low disk space on {{ $labels.alias }} (mountpoint: {{ $labels.mountpoint }})"
+          - name: systemd
+            rules:
+            - alert: FailedUnits
+              expr: node_systemd_unit_state{state="failed"} == 1
+              for: 5m
+              annotations:
+                summary: "service {{ $labels.name }} failed on {{ $labels.alias }}"
+      ''
+    ];
     scrapeConfigs = [
       {
         job_name = "node";
@@ -58,14 +113,13 @@
         };
       }
     ];
-  };
-
-  services.prometheus.exporters = {
-    node = {
-      enable = true;
-      enabledCollectors = ["systemd"];
+    exporters = {
+      node = {
+        enable = true;
+        enabledCollectors = ["systemd"];
+      };
+      systemd.enable = true;
     };
-    systemd.enable = true;
   };
 
   environment.persistence."/persist".directories = ["/var/lib/prometheus2"];

hosts/hopper/profiles/lab/vpn-namespace.nix

@@ -6,6 +6,7 @@
   networking.firewall = let
     allowTcpFromVPNToDefaultPorts = [
       config.services.prometheus.port
+      config.services.prometheus.alertmanager.port
       config.services.vaultwarden.config.ROCKET_PORT
       config.services.homepage-dashboard.listenPort
       config.services.audiobookshelf.port