meshing fixed

mapping.nix

@@ -0,0 +1,19 @@
+# This file is used to map "roles" to nodes
+# Modules are supposed to be generic
+# If some "hostvars" are needed they should
+# be declared here
+{
+    bastion = [
+        "bastion-mtz-lasuite-federez"
+        "bastion-ren-lasuite-federez"
+    ]
+
+    # For instance:
+    # psql = {
+    #     master = "some-node-1";
+    #     slaves = [
+    #         "some-node-2"
+    #         "some-node-3"
+    #     ];
+    # };
+}

secrets/secrets.nix

@@ -57,18 +57,18 @@ in
     "wireguard/wg-private-zone-1-id-13.age".publicKeys = system-wg ++ users;
     
     # Zone 2 (Rennes)
-    "wireguard/wg-private-zone-2-id-1.age".publicKeys = system-wg ++ users;
+    "wireguard/wg-private-zone-2-id-14.age".publicKeys = system-wg ++ users;
-    "wireguard/wg-private-zone-2-id-2.age".publicKeys = system-wg ++ users;
+    "wireguard/wg-private-zone-2-id-15.age".publicKeys = system-wg ++ users;
-    "wireguard/wg-private-zone-2-id-3.age".publicKeys = system-wg ++ users;
+    "wireguard/wg-private-zone-2-id-16.age".publicKeys = system-wg ++ users;
-    "wireguard/wg-private-zone-2-id-4.age".publicKeys = system-wg ++ users;
+    "wireguard/wg-private-zone-2-id-17.age".publicKeys = system-wg ++ users;
-    "wireguard/wg-private-zone-2-id-5.age".publicKeys = system-wg ++ users;
+    "wireguard/wg-private-zone-2-id-18.age".publicKeys = system-wg ++ users;
-    "wireguard/wg-private-zone-2-id-6.age".publicKeys = system-wg ++ users;
+    "wireguard/wg-private-zone-2-id-19.age".publicKeys = system-wg ++ users;
-    "wireguard/wg-private-zone-2-id-7.age".publicKeys = system-wg ++ users;
+    "wireguard/wg-private-zone-2-id-20.age".publicKeys = system-wg ++ users;
-    "wireguard/wg-private-zone-2-id-8.age".publicKeys = system-wg ++ users;
+    "wireguard/wg-private-zone-2-id-21.age".publicKeys = system-wg ++ users;
-    "wireguard/wg-private-zone-2-id-9.age".publicKeys = system-wg ++ users;
+    "wireguard/wg-private-zone-2-id-22.age".publicKeys = system-wg ++ users;
-    "wireguard/wg-private-zone-2-id-10.age".publicKeys = system-wg ++ users;
+    "wireguard/wg-private-zone-2-id-23.age".publicKeys = system-wg ++ users;
-    "wireguard/wg-private-zone-2-id-11.age".publicKeys = system-wg ++ users;
+    "wireguard/wg-private-zone-2-id-24.age".publicKeys = system-wg ++ users;
-    "wireguard/wg-private-zone-2-id-12.age".publicKeys = system-wg ++ users;
+    "wireguard/wg-private-zone-2-id-25.age".publicKeys = system-wg ++ users;
 
     # Bastion secrets
 

shared/commons/mesh.nix

@@ -1,8 +1,14 @@
-{ config, lib, ... }:
+{ config, lib, pkgs, ... }:
 
 let
     # Import nodes
     nodes = import ./../../nodes.nix;
+    myPeer = nodes."${config.hostName}";
+    myId = myPeer.id;
+    myZone = myPeer.zone;
+
+    # And mappings
+    mapping = import ./../../mapping.nix;
 
     buildSecret = zone: id: {
         "wg-private-zone-${toString zone}-id-${toString id}" = {
@@ -13,48 +19,69 @@ let
     };
     generatedSecrets = lib.mapAttrsToList (name: node: buildSecret node.zone node.id) nodes;
 
-    generateWireGuardInterfaces = nodesConfig: let
+    shorten = peerName: let
-        myPeer = nodesConfig."${config.hostName}";
+        parts = lib.splitString "-" peerName;
-        myZone = myPeer.zone;
+        shortened = lib.concatStrings (map (part: lib.substring 0 1 part) parts);
-        myId = myPeer.id;
+    in shortened;
-    
-        # Filter itself out of the peer list
-        peerConfigs = lib.filterAttrs (_peerName: peerConfig: peerConfig.id != myId) nodesConfig;
-    
-        # We'll make one if per peer, this is more flexible
-        interfacePeers = lib.flatten (lib.mapAttrsToList (peerName: peerConfig: let
-            remoteId = peerConfig.id;
 
-            # The mesh is for now only IPv4 based
+    # Filter itself out of the peer list
-            if4 = {
+    peerConfigs = lib.filterAttrs (_peerName: peerConfig: peerConfig.id != myId) nodes;
-                "mesh-${peerName}" = {
+
-                    ips = [
+    # We'll make one if per peer, this is more flexible
-                        "172.19.${toString remoteId}.${toString myId}/32"
+    interfacePeers = lib.flatten (lib.mapAttrsToList (peerName: peerConfig: let
-                        "fc00::${toString remoteId}:${toString myId}/128"
+        remoteId = peerConfig.id;
+        remoteZone = peerConfig.zone;
+
+        # The mesh is for now only IPv4 based
+        if4 = {
+            "mesh-${shorten peerName}-${toString remoteZone}-${toString remoteId}" = {
+                ips = [
+                    "172.19.${toString remoteId}.${toString myId}/32"
+                    "fc00::${toString remoteId}:${toString myId}/128"
+                ];
+                privateKeyFile = config.age.secrets."wg-private-zone-${toString myZone}-id-${toString myId}".path;
+                listenPort = 51000 + remoteId;
+                peers = [{
+                    name = "${peerName}-ip4";
+                    publicKey = peerConfig.wg-pub;
+                    allowedIPs = [
+                        "172.19.${toString myId}.${toString remoteId}/32"
+                        "fc00::${toString myId}:${toString remoteId}/128"
+                        # Allow mgmt transport
+                        "172.19.128.0/17"
+                        "fc00:f::/96"
                     ];
-                    privateKeyFile = config.age.secrets."wg-private-zone-${toString myZone}-id-${toString myId}".path;
+                    persistentKeepalive = 25;
-                    listenPort = 51000 + remoteId;
+                }];
-                    peers = [{
+                # Throw away route created by wireguard
-                        name = "${peerName}-ip4";
+                table = "off";
-                        publicKey = peerConfig.wg-pub;
-                        allowedIPs = [
-                            "172.19.${toString myId}.${toString remoteId}/32"
-                            "fc00::${toString myId}:${toString remoteId}/128"
-                            # Allow mgmt transport
-                            "172.19.128.0/17"
-                            "fc00:f::/96"
-                        ];
-                        persistentKeepalive = 25;
-                    }];
-                };
             };
+        };
 
-        in if4) peerConfigs);
+    in if4) peerConfigs);
 
-        interfaces = builtins.foldl' (acc: set: acc // set) {} interfacePeers;
+    wireguardInterfaces = builtins.foldl' (acc: set: acc // set) {} interfacePeers;
-    in interfaces;
 
-    wireguardInterfaces = generateWireGuardInterfaces nodes;
+    generateRoute = peerName: peerConfig: ''
+        # Static route to declare static wireguard peer inner tunnel ip (that has been thrown away to the "off" table)
+        ${pkgs.iproute2}/bin/ip route replace 172.19.${toString myId}.${toString peerConfig.id} dev mesh-${shorten peerName}-${toString peerConfig.zone}-${toString peerConfig.id} scope link
+        ${pkgs.iproute2}/bin/ip -6 route replace fc00::${toString myId}:${toString peerConfig.id} dev mesh-${shorten peerName}-${toString peerConfig.zone}-${toString peerConfig.id} scope link
+
+        # Return path for mgmt trafic
+        ${if !(lib.elem myPeer mapping.bastion) && (lib.elem peerName mapping.bastion) then ''
+            ${pkgs.iproute2}/bin/ip route replace 172.19.${toString (peerConfig.zone + 127)}.0/24 via 172.19.${toString myPeer.id}.${toString peerConfig.id} dev mesh-${shorten peerName}-${toString peerConfig.zone}-${toString peerConfig.id}
+            ${pkgs.iproute2}/bin/ip -6 route replace fc00:f::${toString (peerConfig.zone + 127)}:0/96 via fc00::${toString myPeer.id}:${toString peerConfig.id} dev mesh-${shorten peerName}-${toString peerConfig.zone}-${toString peerConfig.id}
+        '' else ""}
+    '';
+
+    generateAfter = peerName: peerConfig: "wireguard-mesh-${shorten peerName}-${toString peerConfig.zone}-${toString peerConfig.id}.target";
+
+    routes = lib.mapAttrsToList (name: peer: generateRoute name peer) peerConfigs;
+    afters = lib.mapAttrsToList (name: peer: generateAfter name peer) peerConfigs;
+
+    wireguardStaticRoute = pkgs.writeShellScriptBin "wireguardStaticRoute" ''
+        ${lib.concatStrings routes}
+    '';
 in
 {
     age.secrets = lib.lists.foldl' (acc: set: lib.attrsets.recursiveUpdate acc set) {} generatedSecrets;
@@ -62,17 +89,27 @@ in
     # Networkd backend introduce in 25.05
     # No independant target are generated 
     # when using networkd as a backend
-    # If custom systemd ordering is needed 
+    # We need them, see below
-    # between wg interface and the rest of 
+    networking.wireguard.useNetworkd = false;
-    # networking: switch to false here
-    # Note: if you turn it off it will break
-    # the custom routing for mgmt rt path
-    # trafic handle by networkd, see networking.nix
-    networking.wireguard.useNetworkd = true;
 
     # Return all WireGuard interfaces for each node
     networking.wireguard.interfaces = wireguardInterfaces;
 
+    # Execute custom routing for wireguard
+    systemd.services.wireguardStaticRouting = {
+        wantedBy = [ "multi-user.target" ];
+        # Thus we need specific target to be reached to run it
+        after = afters;
+        description = "Add custom ip route for wireguard.";
+        serviceConfig = {
+            Type = "oneshot";
+            User = "root";
+            ExecStart = "${wireguardStaticRoute}/bin/wireguardStaticRoute";
+        };
+    };
+
     # Open UDP port for wireguard traffic
-    networking.firewall.allowedUDPPorts = lib.range 51000 52000;
+    networking.firewall.allowedUDPPorts = lib.range 51000 (51000 + builtins.head (
+        builtins.sort (a: b: a > b) (
+        lib.mapAttrsToList (name: node: node.id) nodes)));
 }

shared/commons/networking.nix

@@ -15,19 +15,6 @@ let
     # And routes, the gateway is assumed to be in subnet, otherwise GatewayOnLink is required
     route4 = if supportsIPv4 myNode then [{ Gateway = myNode.gIp4; }] else [];
     route6 = if supportsIPv6 myNode then [{ Gateway = myNode.gIp6; }] else [];
-
-    # Return route for mgmt traffic
-    rtwg4 = if myNode.id == 1 then [] else 
-        map (node: { 
-            Gateway = "172.19.${toString node.zone}.1"; 
-            Destination = "172.19.${toString (node.zone + 127)}.0/24"; 
-        }) (lib.attrValues (lib.filterAttrs (name: node: node.id == 1) nodes));
-    
-    rtwg6 = if myNode.id == 1 then [] else
-        map (node: { 
-            Gateway = "fc00::${toString node.zone}:1"; 
-            Destination = "fc00:f::${toString (node.zone + 127)}:0/96"; 
-        }) (lib.attrValues (lib.filterAttrs (name: node: node.id == 1) nodes));
 in
 {
     networking.hostName = config.hostName;
@@ -47,11 +34,6 @@ in
                 # make the routes on this interface a dependency for network-online.target
                 linkConfig.RequiredForOnline = "routable";
             };
-            # This interface is generated by networkd backend for wireguard
-            # See mesh.nix
-            "mesh" = {
-                routes = rtwg4 ++ rtwg6;
-            };
         };
         
         config.addRouteTablesToIPRoute2 = true;