[trigger] Refactorisation (voir détails) - On passe aux tests

* Pour une plus grande modularité, event a été refactorisé, ce qui
 a impliqué de réécrire le fonctionnement des services.
 * Maintenant, y a plus qu'à tester.

gestion/config/trigger.py

@@ -4,6 +4,8 @@
 # Trigger library config file
 # License : GPLv3
 
+import itertools
+
 # Serveur maître
 master = "civet.adm.crans.org"
 
@@ -23,3 +25,8 @@ services = {
             'zamok'     : ["userdel"],
             'zbee'      : ["useradd", "userdel"],
         }
+
+# XXX - Uncomment this when in prod
+#all_services = set([service for service in itertools.chain(*services.values())])
+
+all_services = ['dhcp', 'firewall']

gestion/trigger/host.py

@@ -29,8 +29,8 @@ class TriggerFactory(object):
     def get_services(cls):
         return cls._meths.values()
 
-def record(function):
+def record(cls):
-    TriggerFactory.register(function.func_name, function)
+    TriggerFactory.register(cls.__name__, cls)
 
 def trigger(what):
     return TriggerFactory.get(what)

gestion/trigger/services/dhcp.py

@@ -9,6 +9,7 @@
 
 import lc_ldap.shortcuts
 from gestion.trigger.host import record
+from gestion.trigger.services.service import BasicService
 from cranslib.conffile import ConfFile
 import cranslib.clogger as clogger
 import gestion.config.dhcp as dhcp_config
@@ -16,11 +17,10 @@ import gestion.secrets_new as secrets_new
 import socket
 import gestion.affichage as affichage
 import os
-import sys
 import gestion.iptools as iptools
 
 from gestion.trigger.pypureomapi import pack_ip, pack_mac, OMAPI_OP_UPDATE
-from gestion.trigger.pypureomapi import Omapi, OmapiMessage, OmapiError, OmapiErrorNotFound
+from gestion.trigger.pypureomapi import Omapi, OmapiMessage
 import struct
 
 logger = clogger.CLogger("trigger.dhcp", "debug")
@@ -88,69 +88,107 @@ def lease_clean():
     os.rename(dhcp_config.dhcplease+'.new', dhcp_config.dhcplease)
 
 @record
-def dhcp(body={}):
+class dhcp(BasicService):
-    """Regenerates dhcp service taking body into account.
+    """Class responsible of dhcp service.
 
     """
-    if body and isinstance(body, dict):
+
-        for (mac, ip, name) in body.get("add", []):
+    # Class lookup table to define which changes call which function.
-            add_dhcp_host(mac, ip, name)
+    changes_trigger = {
-        for (mac, ip) in body.get("delete", []):
+        lc_ldap.attributs.macAddress.ldap_name: (dhcp.send_mac_ip,),
-            delete_dhcp_host(mac, ip)
+        lc_ldap.attributs.ipHostNumber.ldap_name: (dhcp.send_mac_ip,),
-        for (rmac, rip, mac, ip, name) in body.get("update", []):
-            delete_dhcp_host(rmac, rip)
-            add_dhcp_host(mac, ip, name)
-    elif body == True:
-        hosts = {}
-        host_template = """
-    host %(nom)s {
-        hardware ethernet %(mac)s;
-        fixed-address %(ip)s;
-        option host-name "%(host)s";
     }
-"""
-        affichage.prettyDoin("Chargement des machines", "...")
-        machines = ldap_conn.allMachines()
-        affichage.prettyDoin("Chargement des machines", "Ok")
-        animation = affichage.Animation(texte="Génération de la configuration",
-                                        nb_cycles=len(machines),
-                                        couleur=True,
-                                        kikoo=True)
 
-        for machine in machines:
+    @classmethod
-            for net in dhcp_config.reseaux.keys():
+    def send_mac_ip(cls, body, diff):
-                ip = str(machine['ipHostNumber'][0])
+        """Computes mac_ip data to send from body and diff
-                mac = str(machine['macAddress'][0])
-                nom = str(machine['host'][0])
-                if '<automatique>' not in [ip, mac] and iptools.AddrInNet(ip, net):
-                    d = {'nom' : nom,
-                         'host' : nom.split(".", 1)[0],
-                         'mac' : mac,
-                         'ip' : ip,
-                        }
-                    try:
-                        hosts[net] += host_template % d
-                    except:
-                        hosts[net] = host_template % d
-            animation.new_step()
-        # Put a \n after the last iteration.
-        animation.end()
 
-        step = "Enregistrement de la configuration dans les fichiers"
+        """
-        affichage.prettyDoin(step, "...")
+        macs = tuple([body[i].get(lc_ldap.attributs.macAddress.ldap_name, [''])[0] for i in xrange(1, 3)])
-        for (net, fichier) in dhcp_config.reseaux.items():
+        ips = tuple([body[i].get(lc_ldap.attributs.ipHostNumber.ldap_name, [''])[0] for i in xrange(1, 3)])
-            with ConfFile(fichier) as configFile:
+        hostnames = tuple([body[i].get(lc_ldap.attributs.host.ldap_name, [''])[0] for i in xrange(1, 3)])
-                configFile.header("#")
-                if hosts.has_key(net):
-                    configFile.write(hosts[net])
-        affichage.prettyDoin(step, "Ok")
 
-        step = "Nettoyage des fichiers de leases"
+        # Régénération du DHCP :
-        affichage.prettyDoin(step, "...")
+        if not macs[0]:
-        try:
+            # Création d'une nouvelle machine.
-            lease_clean()
+            dhcp = {'add': [(macs[1], ips[1], hostnames[1])]}
+        elif not macs[1]:
+            # Destruction d'une machine.
+            dhcp = {'delete': [(macs[0], ips[0])]}
+        else:
+            # Mise à jour.
+            dhcp = {'update': [(macs[0], ips[0], macs[1], ips[1], hostnames[1])]}
+        return ("dhcp", dhcp)
+
+    @classmethod
+    def regen(cls, body=None):
+        """Regenerates dhcp service taking body into account.
+
+        """
+        # http://satyajit.ranjeev.in/2012/01/12/python--dangerous-default-value-as-argument.html
+        # dict are referenced.
+        if body is None:
+            body = {}
+
+        if body and isinstance(body, dict):
+            for (mac, ip, name) in body.get("add", []):
+                add_dhcp_host(mac, ip, name)
+            for (mac, ip) in body.get("delete", []):
+                delete_dhcp_host(mac, ip)
+            for (rmac, rip, mac, ip, name) in body.get("update", []):
+                delete_dhcp_host(rmac, rip)
+                add_dhcp_host(mac, ip, name)
+        elif body == True:
+            hosts = {}
+            host_template = """
+        host %(nom)s {
+            hardware ethernet %(mac)s;
+            fixed-address %(ip)s;
+            option host-name "%(host)s";
+        }
+    """
+            affichage.prettyDoin("Chargement des machines", "...")
+            machines = ldap_conn.allMachines()
+            affichage.prettyDoin("Chargement des machines", "Ok")
+            animation = affichage.Animation(texte="Génération de la configuration",
+                                            nb_cycles=len(machines),
+                                            couleur=True,
+                                            kikoo=True)
+
+            for machine in machines:
+                for net in dhcp_config.reseaux.keys():
+                    ip = str(machine['ipHostNumber'][0])
+                    mac = str(machine['macAddress'][0])
+                    nom = str(machine['host'][0])
+                    if '<automatique>' not in [ip, mac] and iptools.AddrInNet(ip, net):
+                        d = {'nom' : nom,
+                             'host' : nom.split(".", 1)[0],
+                             'mac' : mac,
+                             'ip' : ip,
+                            }
+                        try:
+                            hosts[net] += host_template % d
+                        except:
+                            hosts[net] = host_template % d
+                animation.new_step()
+            # Put a \n after the last iteration.
+            animation.end()
+
+            step = "Enregistrement de la configuration dans les fichiers"
+            affichage.prettyDoin(step, "...")
+            for (net, fichier) in dhcp_config.reseaux.items():
+                with ConfFile(fichier) as configFile:
+                    configFile.header("#")
+                    if hosts.has_key(net):
+                        configFile.write(hosts[net])
             affichage.prettyDoin(step, "Ok")
-        except:
+
-            affichage.prettyDoin(step, "Erreur")
+            step = "Nettoyage des fichiers de leases"
-            print "During lease clean, an error occured."
+            affichage.prettyDoin(step, "...")
-            raise
+            try:
+                lease_clean()
+                affichage.prettyDoin(step, "Ok")
+            except:
+                affichage.prettyDoin(step, "Erreur")
+                print "During lease clean, an error occured."
+                raise

gestion/trigger/services/event.py

@@ -7,16 +7,33 @@
 # License   : GPLv3
 # Date      : 18/05/2014
 
+"""
+This service (event) is designed to receive any modification done on LDAP
+database, and to make a correct diff between former and later object in order
+to guess which services has to be updated.
+"""
+
 import cmb
 import cPickle
-import gestion.config.trigger as trigger_config
-from gestion.trigger.host import record
-import cranslib.clogger as clogger
 import pika
+import importlib
+import itertools
+
+# Trigger features
+import gestion.config.trigger as trigger_config
+from gestion.trigger.host import record, TriggerFactory
+from gestion.trigger.services.service import BasicService
+
+# Clogger
+import cranslib.clogger as clogger
+
+# lc_ldap
 import lc_ldap.attributs
 
 logger = clogger.CLogger("trigger.event", "info")
 
+services = [importlib.import_module("gestion.trigger.services.%s" % (config_service,)) for config_service in trigger_config.all_services]
+
 class Event(cmb.BasicProducer):
     """
     Event tracker
@@ -49,13 +66,16 @@ class Event(cmb.BasicProducer):
             raise
 
     def announce(self, body):
+        """Feature to send message without giving routing_key
+
+        """
         self.send_message("trigger.event", body)
 
 def diff_o_matic(body=()):
     """Fait un diff exhaustif des deux dicos"""
 
     if not body:
-        raise("diff_o_matic received %r as an argument, which is unusable." % (body,))
+        raise ValueError("diff_o_matic received %r as an argument, which is unusable." % (body,))
 
     before = dict(body[1]) or {}
     after = dict(body[2]) or {}
@@ -95,7 +115,6 @@ def compare_lists(list1, list2):
     """
     moins, plus = [], []
 
-    llist2 = [a.lower() for a in list2]
     for elem in [] + list1:
         try:
             ind = list2.index(elem.lower())
@@ -109,67 +128,63 @@ def compare_lists(list1, list2):
     return moins, plus
 
 @record
-def event(body=()):
+class event(BasicService):
-    """Trigger event qui transcrit toute modif ldap en truc exploitable par
+    """Event service class. It extends BasicService, but should not implement
-    trigger. Warning, bootstrap incoming.
+    any change trigger, since it's this service which is designed to call
-
+    change triggers of other services.
-    body est exceptionnellement un tuple. Pour être précis, un 3-tuple.
-    Le premier élément est le dn de l'objet LDAP, il est pas indispensable.
-    Le deuxième est un dico qui recense l'état complet de l'objet modifié avant
-    validation des modifications.
-    Le troisième est un dico qui recense l'état complet de l'objet modifié après
-    modification.
-
-    Si l'objet vient d'être créé, le deuxième élément est None.
-    Si l'objet vient d'être supprimé, le troisième élément vaut None.
-
-    Il faut donc faire un diff, générer la liste des triggers à envoyer, puis
-    les envoyer.
 
     """
 
-    logger.info("Received message %r…", body)
+    @classmethod
+    def get_changes(cls, body, diff):
+        """Compute changes from diff"""
 
-    diff = diff_o_matic(body)
+        return [None]
 
-    # À cette étape, on a un dico des attrs ayant subi une modif
+    @classmethod
-    # a["macAddress"] par exemple, pourrait ressembler à
+    def regen(cls, body=()):
-    # (["aa:bb:cc:dd:ee:fg"], ["aa:bb:cc:dd:ee:ff"]), la liste de gauche
+        """When any event arrives on trigger-civet-event, this method is called
-    # étant les trucs perdus, celle de droite ceux gagnés. Suivant le type
+        and designed to transcript the body (ldap data) in something usable for
-    # des attributs, ça peut être un remplacement (mac, ip...), ou juste
+        the services. Afterwards, it sends these transcripts on the good way
-    # des retraits/ajouts (mailAlias...)
+        using routing_key.
-    # Avec ça on peut trigger tout ce qu'on veut.
 
-    # Si la mac ou l'IP a changé…
+        body is a 3-tuple, containing LDAP dn, the former state of the object
-    if diff.has_key(lc_ldap.attributs.ipHostNumber.ldap_name) or diff.has_key(lc_ldap.attributs.macAddress.ldap_name):
+        (a simple dict), and the later state. The data are non-binding-dependant.
-        logger.info("Detected MAC or IP update, calling trigger_mac_ip…")
-        trigger_mac_ip(body, diff)
 
-def trigger_mac_ip(body, diff):
+        A new object has body[1] to None, a deleted one has body[2] to None.
-    macs      = tuple([body[i].get(lc_ldap.attributs.macAddress.ldap_name, [''])[0] for i in xrange(1, 3)])
-    ips       = tuple([body[i].get(lc_ldap.attributs.ipHostNumber.ldap_name, [''])[0] for i in xrange(1, 3)])
-    hostnames = tuple([body[i].get(lc_ldap.attributs.host.ldap_name, [''])[0] for i in xrange(1, 3)])
 
-    # Régénération du DHCP :
+        """
-    if not macs[0]:
+
-        # Création d'une nouvelle machine.
+        logger.info("Received message %r…", body)
-        dhcp = {'add': [(macs[1], ips[1], hostnames[1])]}
+
-        fw = {'add': [(macs[1], ips[1])]}
+        diff = diff_o_matic(body)
-    elif not macs[1]:
+
-        # Destruction d'une machine.
+        # Now, diff is a dict containing attributes which has been modified.
-        dhcp = {'delete': [(macs[0], ips[0])]}
+        # diff['macAddress'] could look like (['aa:bb:cc:dd:ee:fg'], ['aa:bb:cc:dd:ee:ff']),
-        fw = {'delete': [(macs[0], ips[0])]}
+        # where the list on the left is the former value of attributes, and the list on the
-    else:
+        # right the latter values.
-        # Mise à jour.
+
-        dhcp = {'update': [(macs[0], ips[0], macs[1], ips[1], hostnames[1])]}
+        #        -*- Explain -*-
-        fw = {'update': [(macs[0], ips[0], macs[1], ips[1])]}
+        #In [11]: import itertools
-    logger.info("Sending DHCP trigger with body %r", dhcp)
+        #
-    # XXX - Remove # when putting in production, needs further tests
+        #In [12]: a = [[(3, 'lol'), ('7', 3)], [(5, 6), None], [None], [('lol', 'lal')]]
-    #trigger_send("dhcp", dhcp)
+        #
-    logger.info("Sending firewall trigger for mac_ip with body %r", fw)
+        #In [13]: a
-    # XXX - Remove # when in prod, tested on 15/06/2014, functionnal.
+        #Out[13]: [[(3, 'lol'), ('7', 3)], [(5, 6), None], [None], [('lol', 'lal')]]
-    trigger_send("firewall", ("mac_ip", fw))
+        #
-    logger.info("trigger_mac_ip done.")
+        #In [14]: list(set([message for message in itertools.chain(*a)]))
+        #Out[14]: [('7', 3), (5, 6), None, ('lol', 'lal'), (3, 'lol')] # Only one None from a, since [None, x, y, None] is equivalent for itertools to [x, y]
+        #
+        #In [15]: b = list(set([message for message in itertools.chain(*a) if message is not None]))
+        #
+        #In [16]: b
+        #Out[16]: [('7', 3), (5, 6), ('lol', 'lal'), (3, 'lol')]
+        msg_to_send = [message for message in itertools.chain(*[service.get_changes(body, diff) for service in TriggerFactory.get_services()]) if message is not None]
+
+        for msg in msg_to_send:
+            logger.info("Sending %r on the road \\o/", msg)
+            # XXX - uncomment this when in production
+            # trigger_send(*msg)
 
 def trigger_send(routing_key, body):
     sender = Event("civet")

gestion/trigger/services/firewall.py

@@ -8,14 +8,18 @@
 # Author  : Pierre-Elliott Bécue <becue@crans.org>
 # Licence : GPLv3
 # Date    : 15/06/2014
+"""
+Firewall service module. is uses the firewall library as it's, it
+is not designed to replace it, just to call specific functions from
+it to regenerate what needs to.
+"""
 
 import lc_ldap.shortcuts
 from gestion.trigger.host import record
+from gestion.trigger.services.service import BasicService
 import cranslib.clogger as clogger
 import gestion.config.firewall as firewall_config
 import gestion.trigger.firewall4.firewall4 as firewall4
-import os
-import sys
 
 logger = clogger.CLogger("trigger.firewall", "debug")
 
@@ -28,25 +32,73 @@ class FwFunFactory(object):
 
     @classmethod
     def register(cls, key, value):
+        """Stores in factory the function name and its value
+
+        """
         cls._meths[key] = value
 
     @classmethod
     def get(cls, key):
+        """Gets what is stored
+
+        """
         return cls._meths.get(key, None)
 
 def fwrecord(function):
+    """Records function in FwFunFactory
+
+    """
     FwFunFactory.register(function.func_name, function)
 
 def fwcall(fwfun):
+    """Calls in function from FwFunFactory
+
+    """
     return FwFunFactory.get(fwfun)
 
 @record
-def firewall(body=()):
+class firewall(BasicService):
-    if len(body) != 2:
+    """Firewall service that handles any modification in the firewall.
-        logger.warning("Received body %r, this format is incorrect, discarding.", body)
+
-    (service, data) = body
+    """
-    logger.info("Calling service %s for data %r", service, data)
+
-    fwcall(service)(data)
+    # Class lookup table to define which changes call which function.
+    changes_trigger = {
+        lc_ldap.attributs.macAddress.ldap_name: (firewall.send_mac_ip,),
+        lc_ldap.attributs.ipHostNumber.ldap_name: (firewall.send_mac_ip,),
+    }
+
+    @classmethod
+    def send_mac_ip(cls, body, diff):
+        """Computes mac_ip data to send from body and diff
+
+        """
+        macs = tuple([body[i].get(lc_ldap.attributs.macAddress.ldap_name, [''])[0] for i in xrange(1, 3)])
+        ips = tuple([body[i].get(lc_ldap.attributs.ipHostNumber.ldap_name, [''])[0] for i in xrange(1, 3)])
+
+        # Mise à jour du parefeu mac_ip
+        if not macs[0]:
+            # Création d'une nouvelle machine.
+            fw = {'add': [(macs[1], ips[1])]}
+        elif not macs[1]:
+            # Destruction d'une machine.
+            fw = {'delete': [(macs[0], ips[0])]}
+        else:
+            # Mise à jour.
+            fw = {'update': [(macs[0], ips[0], macs[1], ips[1])]}
+        return ("firewall", ("mac_ip", fw))
+
+    @classmethod
+    def regen(cls, body=()):
+        """Regens the specific service
+
+        """
+        if len(body) != 2:
+            logger.warning("Received body %r, this format is incorrect, discarding.", body)
+            return
+        (service, data) = body
+        logger.info("Calling service %s for data %r", service, data)
+        fwcall(service)(data)
 
 @fwrecord
 def mac_ip(body):

gestion/trigger/services/service.py

@@ -0,0 +1,38 @@
+#!/bin/bash /usr/scripts/python.sh
+# -*- coding: utf-8 -*-
+
+"""
+This module provides a basic service class to other services. It should *NOT*
+be referenced in configuration of trigger.
+"""
+
+class BasicService(object):
+    """Basic service handler. Other services should inherit fron this one.
+
+    """
+
+    changes_trigger = {}
+
+    @classmethod
+    def get_changes(cls, body, diff):
+        """Looks for changes and creates messages to send back
+
+        """
+        # list of all messages to send.
+        msg_list = []
+
+        # lists all functions to call
+        func_list = set()
+        for (attrib, functions) in cls.changes_trigger.iteritems():
+            if attrib in diff:
+                func_list.update(functions)
+        for function in func_list:
+            msg_list.append(function(body, diff))
+        return msg_list
+
+    @classmethod
+    def regen(cls, body):
+        """This method is referenced to avoid uncaught exceptions
+
+        """
+        pass

gestion/trigger/trigger.py

@@ -62,7 +62,10 @@ class EvenementListener(cmb.AsynchronousConsumer):
         # On tente d'invoquer le trigger attendu, à l'aide de la méthode trigger
         # about contient le nom de la fonction à appeler, body lui est filé en argument.
         try:
-            trigger(about)(body)
+            if about in trigger_config.services[hostname]:
+                trigger(about).regen(body)
+            else:
+                raise AttributeError
         except AttributeError:
             logger.warning('No suitable trigger found for message # %s from %s: %s on host %s. Discarding it.',
                            basic_deliver.delivery_tag, properties.app_id, body, hostname)