Merge branch 'master' into add_gamma

profiles/backups.nix

@@ -6,10 +6,9 @@ let
   postgresql = config.services.postgresql.package;
   additionalPackages = [
     pkgs.coreutils
-    postgresql
     pkgs.sudo
     pkgs.sqlite
-  ];
+  ] ++ lib.optionals (builtins.length cfg.postgresqlDatabases > 0) [ postgresql ];
   remotes = {
     memoragon = {
       host = "memoragon.infra.federez.net";

profiles/grafana.nix

@@ -86,7 +86,9 @@ in {
       [
         {
           name = "VictoriaMetrics";
-          type = "victoriametrics-metrics-datasource";
+          # See https://github.com/VictoriaMetrics/victoriametrics-datasource/issues/59
+          type = "prometheus";
+          #type = "victoriametrics-metrics-datasource";
           uid = "vm";
           url = "http://localhost:8428";
           editable = false;

profiles/monitoring/default.nix

@@ -5,6 +5,7 @@ let
   alertmanagerPort = config.services.prometheus.alertmanager.port;
   alertbotPort = config.services.alertbot.listenPort;
   blackboxPort = config.services.prometheus.exporters.blackbox.port;
+  vmalertPort = 8880;
   nodePort = 9100;
   mkScrapeConfig = name: config: {
     job_name = name;
@@ -30,6 +31,7 @@ let
       (alert: attrs: attrs // { inherit alert; })
       (import path { inherit critical warning; });
   });
+  inactiveNodes = [ "wagon" "jargon" "lagon" ];
 in {
   imports = [
     ../../modules/alertbot.nix
@@ -41,14 +43,22 @@ in {
 
   services.victoriametrics = {
     enable = true;
-    extraOptions = [ "-enableTCP6" ];
+    extraOptions = [
+      "-enableTCP6"
+      "-vmalert.proxyURL=http://localhost:${toString vmalertPort}"
+    ];
     listenAddress = "localhost:${toString victoriametricsPort}";
     prometheusConfig = {
       scrape_configs = mkScrapeConfigs {
         node = {
           path = "/metrics";
           replacement = "$1.infra.federez.net:${toString nodePort}";
-          targets = lib.attrsets.mapAttrsToList (n: _: n) network.infra.nodes;
+          targets = let
+            activeNodes = lib.filterAttrs
+              (n: _: !(builtins.elem n inactiveNodes))
+              network.infra.nodes;
+          in
+            lib.attrsets.mapAttrsToList (n: _: n) activeNodes;
         };
         blackbox_https_get_200 = {
           path = "/probe";
@@ -90,6 +100,7 @@ in {
       "remoteWrite.url" = victoriametricsUrl;
       "remoteRead.url" = victoriametricsUrl;
       "notifier.url" = [ alertmanagerUrl ];
+      "httpListenAddr" = "localhost:${toString vmalertPort}";
     };
   };
 

profiles/monitoring/rules/node.nix

@@ -131,7 +131,7 @@
 
   NodePhysicalComponentTooHot = {
     expr = ''
-      node_hwmon_temp_celsius > clamp_max(79, node_hwmon_temp_max_celsius)
+      node_hwmon_temp_celsius > clamp_max(node_hwmon_temp_max_celsius, 79)
     '';
     for = "0m";
     labels = critical;
@@ -265,7 +265,7 @@
   NodeLoad5Usage = {
     expr = ''
       node_load5 / (
-        count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})) > 1.1
+        count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})) > 1.25
     '';
     for = "1m";
     labels = warning;

profiles/vogon/host.nix

@@ -101,7 +101,7 @@
 
   age.secrets = {
     vogon-wg-infra-key = {
-      file = ../secrets/vogon-wg-infra-key.age;
+      file = ../../secrets/vogon-wg-infra-key.age;
       owner = "root";
       group = "root";
     };