more metrics for prometheus

hosts/hopper/profiles/lab/caddy.nix

@@ -21,7 +21,9 @@ in {
   services.caddy = {
     enable = true;
     globalConfig = ''
-      metrics
+      metrics {
+        per_host
+      }
       servers {
         trusted_proxies static 10.0.0.1
       }

hosts/hopper/profiles/lab/prometheus.nix

@@ -10,9 +10,16 @@
     scrapeConfigs = [
       {
         job_name = "node";
-        static_configs = lib.singleton {
+        static_configs = [
-          targets = ["127.0.0.1:${toString config.services.prometheus.exporters.node.port}"];
+          {
-        };
+            targets = ["127.0.0.1:9100"];
+            labels.alias = "hopper";
+          }
+          {
+            targets = ["rackserv:9100"];
+            labels.alias = "rackserv";
+          }
+        ];
       }
       {
         job_name = "tailscale_client";
@@ -22,9 +29,16 @@
       }
       {
         job_name = "caddy";
-        static_configs = lib.singleton {
+        static_configs = [
+          {
             targets = ["${config.vpnNamespaces."wg".namespaceAddress}:2019"];
-        };
+            labels.alias = "hopper";
+          }
+          {
+            targets = ["rackserv:2019"];
+            labels.alias = "rackserv";
+          }
+        ];
       }
       {
         job_name = "slskd";
@@ -37,6 +51,12 @@
           action = "drop";
         };
       }
+      {
+        job_name = "forgejo";
+        static_configs = lib.singleton {
+          targets = ["rackserv:9615"];
+        };
+      }
     ];
   };
 

hosts/rackserv/default.nix

@@ -14,6 +14,7 @@
       ./profiles/backups.nix
       ./profiles/caddy.nix
       ./profiles/forgejo.nix
+      ./profiles/prometheus.nix
     ]
     ++ (with systemProfiles; [
       core.security

hosts/rackserv/profiles/caddy.nix

@@ -22,9 +22,20 @@ in {
     };
   };
 
+  # systemd.services.caddy.environment.CADDY_ADMIN = "${vars.tailnet.rackserv}:2019";
   services.caddy = {
     enable = true;
-    virtualHosts = {
+    globalConfig = ''
+      metrics {
+        per_host
+      }
+      admin :2019 {
+        origins 127.0.0.1 100.64.0.0/10
+      }
+    '';
+    virtualHosts = let
+      forgejoPort = toString config.services.forgejo.settings.server.HTTP_PORT;
+    in {
       misc = {
         hostName = "${domain}";
         serverAliases = ["*.${domain}"];
@@ -37,12 +48,21 @@ in {
         hostName = "git.${domain}";
         useACMEHost = domain;
         extraConfig = ''
-          reverse_proxy localhost:${toString config.services.forgejo.settings.server.HTTP_PORT}
+          respond /metrics 403
+          reverse_proxy localhost:${forgejoPort}
         '';
       };
-      other = {
+      forgejoMetrics = {
+        hostName = ":9615";
         extraConfig = ''
-          respond 404
+          @blocked {
+            not {
+              client_ip ${vars.tailnet.hopper}
+              path /metrics
+            }
+          }
+          respond @blocked 403
+          reverse_proxy localhost:${forgejoPort}
         '';
       };
     };

hosts/rackserv/profiles/forgejo.nix

@@ -11,6 +11,11 @@
         HTTP_PORT = 3000;
         HTTP_ADDR = "127.0.0.1";
       };
+      metrics = {
+        ENABLED = true;
+        ENABLED_ISSUE_BY_LABEL = true;
+        ENABLED_ISSUE_BY_REPOSITORY = true;
+      };
       service.DISABLE_REGISTRATION = true;
     };
   };

hosts/rackserv/profiles/prometheus.nix

@@ -0,0 +1,8 @@
+{
+  services.prometheus.exporters = {
+    node = {
+      enable = true;
+      enabledCollectors = ["systemd"];
+    };
+  };
+}