monitoring: add support for Alert list visualisation in Grafana

This commit is contained in:
jeltz 2025-07-06 23:02:21 +02:00
parent a1c4879a38
commit 4e6513466f
Signed by: jeltz
GPG key ID: 800882B66C0C3326
2 changed files with 16 additions and 3 deletions

View file

@ -86,7 +86,9 @@ in {
[ [
{ {
name = "VictoriaMetrics"; name = "VictoriaMetrics";
type = "victoriametrics-metrics-datasource"; # See https://github.com/VictoriaMetrics/victoriametrics-datasource/issues/59
type = "prometheus";
#type = "victoriametrics-metrics-datasource";
uid = "vm"; uid = "vm";
url = "http://localhost:8428"; url = "http://localhost:8428";
editable = false; editable = false;

View file

@ -5,6 +5,7 @@ let
alertmanagerPort = config.services.prometheus.alertmanager.port; alertmanagerPort = config.services.prometheus.alertmanager.port;
alertbotPort = config.services.alertbot.listenPort; alertbotPort = config.services.alertbot.listenPort;
blackboxPort = config.services.prometheus.exporters.blackbox.port; blackboxPort = config.services.prometheus.exporters.blackbox.port;
vmalertPort = 8880;
nodePort = 9100; nodePort = 9100;
mkScrapeConfig = name: config: { mkScrapeConfig = name: config: {
job_name = name; job_name = name;
@ -30,6 +31,7 @@ let
(alert: attrs: attrs // { inherit alert; }) (alert: attrs: attrs // { inherit alert; })
(import path { inherit critical warning; }); (import path { inherit critical warning; });
}); });
inactiveNodes = [ "wagon" "jargon" "lagon" ];
in { in {
imports = [ imports = [
../../modules/alertbot.nix ../../modules/alertbot.nix
@ -41,14 +43,22 @@ in {
services.victoriametrics = { services.victoriametrics = {
enable = true; enable = true;
extraOptions = [ "-enableTCP6" ]; extraOptions = [
"-enableTCP6"
"-vmalert.proxyURL=http://localhost:${toString vmalertPort}"
];
listenAddress = "localhost:${toString victoriametricsPort}"; listenAddress = "localhost:${toString victoriametricsPort}";
prometheusConfig = { prometheusConfig = {
scrape_configs = mkScrapeConfigs { scrape_configs = mkScrapeConfigs {
node = { node = {
path = "/metrics"; path = "/metrics";
replacement = "$1.infra.federez.net:${toString nodePort}"; replacement = "$1.infra.federez.net:${toString nodePort}";
targets = lib.attrsets.mapAttrsToList (n: _: n) network.infra.nodes; targets = let
activeNodes = lib.filterAttrs
(n: _: !(builtins.elem n inactiveNodes))
network.infra.nodes;
in
lib.attrsets.mapAttrsToList (n: _: n) activeNodes;
}; };
blackbox_https_get_200 = { blackbox_https_get_200 = {
path = "/probe"; path = "/probe";
@ -90,6 +100,7 @@ in {
"remoteWrite.url" = victoriametricsUrl; "remoteWrite.url" = victoriametricsUrl;
"remoteRead.url" = victoriametricsUrl; "remoteRead.url" = victoriametricsUrl;
"notifier.url" = [ alertmanagerUrl ]; "notifier.url" = [ alertmanagerUrl ];
"httpListenAddr" = "localhost:${toString vmalertPort}";
}; };
}; };