monitoring: add support for Alert list visualisation in Grafana

profiles/grafana.nix

@@ -86,7 +86,9 @@ in {
       [
         {
           name = "VictoriaMetrics";
-          type = "victoriametrics-metrics-datasource";
+          # See https://github.com/VictoriaMetrics/victoriametrics-datasource/issues/59
+          type = "prometheus";
+          #type = "victoriametrics-metrics-datasource";
           uid = "vm";
           url = "http://localhost:8428";
           editable = false;

profiles/monitoring/default.nix

@@ -5,6 +5,7 @@ let
   alertmanagerPort = config.services.prometheus.alertmanager.port;
   alertbotPort = config.services.alertbot.listenPort;
   blackboxPort = config.services.prometheus.exporters.blackbox.port;
+  vmalertPort = 8880;
   nodePort = 9100;
   mkScrapeConfig = name: config: {
     job_name = name;
@@ -30,6 +31,7 @@ let
       (alert: attrs: attrs // { inherit alert; })
       (import path { inherit critical warning; });
   });
+  inactiveNodes = [ "wagon" "jargon" "lagon" ];
 in {
   imports = [
     ../../modules/alertbot.nix
@@ -41,14 +43,22 @@ in {
 
   services.victoriametrics = {
     enable = true;
-    extraOptions = [ "-enableTCP6" ];
+    extraOptions = [
+      "-enableTCP6"
+      "-vmalert.proxyURL=http://localhost:${toString vmalertPort}"
+    ];
     listenAddress = "localhost:${toString victoriametricsPort}";
     prometheusConfig = {
       scrape_configs = mkScrapeConfigs {
         node = {
           path = "/metrics";
           replacement = "$1.infra.federez.net:${toString nodePort}";
-          targets = lib.attrsets.mapAttrsToList (n: _: n) network.infra.nodes;
+          targets = let
+            activeNodes = lib.filterAttrs
+              (n: _: !(builtins.elem n inactiveNodes))
+              network.infra.nodes;
+          in
+            lib.attrsets.mapAttrsToList (n: _: n) activeNodes;
         };
         blackbox_https_get_200 = {
           path = "/probe";
@@ -90,6 +100,7 @@ in {
       "remoteWrite.url" = victoriametricsUrl;
       "remoteRead.url" = victoriametricsUrl;
       "notifier.url" = [ alertmanagerUrl ];
+      "httpListenAddr" = "localhost:${toString vmalertPort}";
     };
   };