From a52ef72a5461f2f63501649257072c6ade5f7641 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Elliott=20B=C3=A9cue?= Date: Mon, 14 Jul 2014 03:10:42 +0200 Subject: [PATCH] =?UTF-8?q?[trigger]=20MetaService=20en=20place,=20et=20am?= =?UTF-8?q?=C3=A9lioration=20du=20d=C3=A9bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cranslib/clogger.py | 13 ++- gestion/config/trigger.py | 2 + gestion/trigger/host.py | 2 +- gestion/trigger/services/dhcp.py | 167 +++++++++++++++------------ gestion/trigger/services/event.py | 20 ++-- gestion/trigger/services/firewall.py | 75 ++++-------- gestion/trigger/services/service.py | 54 +++++++++ gestion/trigger/trigger.py | 15 ++- 8 files changed, 202 insertions(+), 146 deletions(-) diff --git a/cranslib/clogger.py b/cranslib/clogger.py index 09a0f85a..1672c913 100644 --- a/cranslib/clogger.py +++ b/cranslib/clogger.py @@ -13,9 +13,9 @@ class CLogger(logging.Logger): Crans logger """ - def __init__(self, loggerName, level): + def __init__(self, loggerName, service, level, debug=False): """ - Initialise le logger + Initializes logger. The debug variable is useful to have a print to stdout (when debugging) """ super(CLogger, self).__init__(loggerName) @@ -27,10 +27,17 @@ class CLogger(logging.Logger): self.fh.setLevel(self.fhlevel) # Creates formatter - self.formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + self.formatter = logging.Formatter('%%(asctime)s - %%(name)s - %(service)s - %%(levelname)s - %%(message)s' % {'service': service}) # Adds formatter to FileHandler self.fh.setFormatter(self.formatter) + if debug: + self.sh = logging.StreamHandler() + self.shlevel = logging.DEBUG + self.sh.setLevel(self.shlevel) + self.sh.setFormatter(self.formatter) + self.addHandler(self.sh) + # Adds FileHandler to Handlers self.addHandler(self.fh) diff --git a/gestion/config/trigger.py b/gestion/config/trigger.py index b295c3a4..314a9bb7 100644 --- a/gestion/config/trigger.py +++ b/gestion/config/trigger.py @@ -6,6 +6,8 @@ import itertools +debug = True + # Serveur maître master = "civet.adm.crans.org" diff --git a/gestion/trigger/host.py b/gestion/trigger/host.py index 9d8579ed..067c66e9 100644 --- a/gestion/trigger/host.py +++ b/gestion/trigger/host.py @@ -30,7 +30,7 @@ class TriggerFactory(object): return cls._meths.values() def record(cls): - TriggerFactory.register(cls.__name__, cls) + TriggerFactory.register(cls.__name__.lower(), cls) def trigger(what): return TriggerFactory.get(what) diff --git a/gestion/trigger/services/dhcp.py b/gestion/trigger/services/dhcp.py index f30158fb..3cec2516 100644 --- a/gestion/trigger/services/dhcp.py +++ b/gestion/trigger/services/dhcp.py @@ -8,7 +8,7 @@ # Licence : GPLv3 import lc_ldap.shortcuts -from gestion.trigger.host import record +import gestion.config.trigger as trigger_config from gestion.trigger.services.service import BasicService from cranslib.conffile import ConfFile import cranslib.clogger as clogger @@ -23,82 +23,25 @@ from gestion.trigger.pypureomapi import pack_ip, pack_mac, OMAPI_OP_UPDATE from gestion.trigger.pypureomapi import Omapi, OmapiMessage import struct -logger = clogger.CLogger("trigger.dhcp", "debug") +logger = clogger.CLogger("trigger", "dhcp", "debug", trigger_config.debug) hostname = socket.gethostname().split(".")[0] + ".adm.crans.org" -dhcp_omapi_keyname = secrets_new.get("dhcp_omapi_keyname") -dhcp_omapi_key = secrets_new.get("dhcp_omapi_keys")[hostname] ldap_conn = lc_ldap.shortcuts.lc_ldap_readonly() -def add_dhcp_host(mac, ip, name=None): - """Adds a dhcp host using omapi - """ - - if '' in [ip, mac]: - return - msg = OmapiMessage.open(b"host") - msg.message.append((b"create", struct.pack("!I", 1))) - msg.message.append((b"exclusive", struct.pack("!I", 1))) - msg.obj.append((b"hardware-address", pack_mac(mac))) - msg.obj.append((b"hardware-type", struct.pack("!I", 1))) - msg.obj.append((b"ip-address", pack_ip(ip))) - if name: - msg.obj.append((b"name", bytes(name))) - conn = Omapi(hostname, 9991, dhcp_omapi_keyname, dhcp_omapi_key) - response = conn.query_server(msg) - conn.close() - -def delete_dhcp_host(mac, ip): - """Deletes dhcp host using omapi - - """ - - if '' in [ip, mac]: - return - msg = OmapiMessage.open(b"host") - msg.obj.append((b"hardware-address", pack_mac(mac))) - msg.obj.append((b"hardware-type", struct.pack("!I", 1))) - msg.obj.append((b"ip-address", pack_ip(ip))) - conn = Omapi(hostname, 9991, dhcp_omapi_keyname, dhcp_omapi_key) - response = conn.query_server(msg) - if response.opcode == OMAPI_OP_UPDATE: - response = conn.query_server(OmapiMessage.delete(response.handle)) - conn.close() - -def lease_clean(): - """Clean the lease file - - """ - # TODO : use ConfigFile structure - leasefile = open(dhcp_config.dhcplease) - newleasefile = open(dhcp_config.dhcplease + '.new', 'w') - config = "" - line = leasefile.readline() - write = True - while line: - if line.strip().startswith('host'): - write = False - if write: - newleasefile.write(line) - if not write and line.strip().endswith('}'): - write = True - line = leasefile.readline() - leasefile.close() - newleasefile.close() - os.rename(dhcp_config.dhcplease+'.new', dhcp_config.dhcplease) - -@record -class dhcp(BasicService): +class Dhcp(BasicService): """Class responsible of dhcp service. """ # Class lookup table to define which changes call which function. changes_trigger = { - lc_ldap.attributs.macAddress.ldap_name: (dhcp.send_mac_ip,), - lc_ldap.attributs.ipHostNumber.ldap_name: (dhcp.send_mac_ip,), + lc_ldap.attributs.macAddress.ldap_name: ('send_mac_ip',), + lc_ldap.attributs.ipHostNumber.ldap_name: ('send_mac_ip',), } + dhcp_omapi_keyname = None + dhcp_omapi_key = None #secrets_new.get("dhcp_omapi_keys")[hostname] + @classmethod def send_mac_ip(cls, body, diff): """Computes mac_ip data to send from body and diff @@ -111,14 +54,14 @@ class dhcp(BasicService): # Régénération du DHCP : if not macs[0]: # Création d'une nouvelle machine. - dhcp = {'add': [(macs[1], ips[1], hostnames[1])]} + dhcp_dict = {'add': [(macs[1], ips[1], hostnames[1])]} elif not macs[1]: # Destruction d'une machine. - dhcp = {'delete': [(macs[0], ips[0])]} + dhcp_dict = {'delete': [(macs[0], ips[0])]} else: # Mise à jour. - dhcp = {'update': [(macs[0], ips[0], macs[1], ips[1], hostnames[1])]} - return ("dhcp", dhcp) + dhcp_dict = {'update': [(macs[0], ips[0], macs[1], ips[1], hostnames[1])]} + return ("dhcp", dhcp_dict) @classmethod def regen(cls, body=None): @@ -132,12 +75,12 @@ class dhcp(BasicService): if body and isinstance(body, dict): for (mac, ip, name) in body.get("add", []): - add_dhcp_host(mac, ip, name) + cls.add_dhcp_host(mac, ip, name) for (mac, ip) in body.get("delete", []): - delete_dhcp_host(mac, ip) + cls.delete_dhcp_host(mac, ip) for (rmac, rip, mac, ip, name) in body.get("update", []): - delete_dhcp_host(rmac, rip) - add_dhcp_host(mac, ip, name) + cls.delete_dhcp_host(rmac, rip) + cls.add_dhcp_host(mac, ip, name) elif body == True: hosts = {} host_template = """ @@ -186,9 +129,85 @@ class dhcp(BasicService): step = "Nettoyage des fichiers de leases" affichage.prettyDoin(step, "...") try: - lease_clean() + cls.lease_clean() affichage.prettyDoin(step, "Ok") except: affichage.prettyDoin(step, "Erreur") print "During lease clean, an error occured." raise + + @classmethod + def check_secrets(cls): + """This method allows lazy evaluation for dhcp_omapi_keyname + and dhcp_omapi_key, since event imports all services. This is actually + the best lazy eval we can hope, since property won't work on + classmethods. + + """ + if cls.dhcp_omapi_keyname is None: + cls.dhcp_omapi_keyname = secrets_new.get("dhcp_omapi_keyname") + if cls.dhcp_omapi_key is None: + cls.dhcp_omapi_key = secrets_new.get("dhcp_omapi_keys")[hostname] + + @classmethod + def add_dhcp_host(cls, mac, ip, name=None): + """Adds a dhcp host using omapi + + """ + cls.check_secrets() + + if '' in [ip, mac]: + return + msg = OmapiMessage.open(b"host") + msg.message.append((b"create", struct.pack("!I", 1))) + msg.message.append((b"exclusive", struct.pack("!I", 1))) + msg.obj.append((b"hardware-address", pack_mac(mac))) + msg.obj.append((b"hardware-type", struct.pack("!I", 1))) + msg.obj.append((b"ip-address", pack_ip(ip))) + if name: + msg.obj.append((b"name", bytes(name))) + conn = Omapi(hostname, 9991, cls.dhcp_omapi_keyname, cls.dhcp_omapi_key) + _ = conn.query_server(msg) + conn.close() + + @classmethod + def delete_dhcp_host(cls, mac, ip): + """Deletes dhcp host using omapi + + """ + cls.check_secrets() + + if '' in [ip, mac]: + return + msg = OmapiMessage.open(b"host") + msg.obj.append((b"hardware-address", pack_mac(mac))) + msg.obj.append((b"hardware-type", struct.pack("!I", 1))) + msg.obj.append((b"ip-address", pack_ip(ip))) + conn = Omapi(hostname, 9991, cls.dhcp_omapi_keyname, cls.dhcp_omapi_key) + response = conn.query_server(msg) + if response.opcode == OMAPI_OP_UPDATE: + _ = conn.query_server(OmapiMessage.delete(response.handle)) + conn.close() + + @staticmethod + def lease_clean(): + """Clean the lease file + + """ + # TODO : use ConfigFile structure + leasefile = open(dhcp_config.dhcplease) + newleasefile = open(dhcp_config.dhcplease + '.new', 'w') + line = leasefile.readline() + write = True + while line: + if line.strip().startswith('host'): + write = False + if write: + newleasefile.write(line) + if not write and line.strip().endswith('}'): + write = True + line = leasefile.readline() + leasefile.close() + newleasefile.close() + os.rename(dhcp_config.dhcplease+'.new', dhcp_config.dhcplease) + diff --git a/gestion/trigger/services/event.py b/gestion/trigger/services/event.py index 5a848cac..4cbc3add 100644 --- a/gestion/trigger/services/event.py +++ b/gestion/trigger/services/event.py @@ -18,6 +18,7 @@ import cPickle import pika import importlib import itertools +import traceback # Trigger features import gestion.config.trigger as trigger_config @@ -30,20 +31,24 @@ import cranslib.clogger as clogger # lc_ldap import lc_ldap.attributs -logger = clogger.CLogger("trigger.event", "info") +logger = clogger.CLogger("trigger", "event", "debug", trigger_config.debug) -services = [importlib.import_module("gestion.trigger.services.%s" % (config_service,)) for config_service in trigger_config.all_services] +for config_service in trigger_config.all_services: + try: + services.append(importlib.import_module("gestion.trigger.services.%s" % (config_service,))) + except Exception as e: + logger.critical("Fatal : import of %r failed, see following traceback. %r", config_service, traceback.format_exc()) -class Event(cmb.BasicProducer): +class EventProducer(cmb.BasicProducer): """ - Event tracker + EventProducer tracker """ def __init__(self, app_id): """Extended """ - logger.info("Starting trigger Event program…") - super(Event, self).__init__(trigger_config.master, 'trigger', app_id) + logger.info("Starting trigger EventProducer program…") + super(EventProducer, self).__init__(trigger_config.master, 'trigger', app_id) self._connection = self.connect() self.get_chan() @@ -127,7 +132,6 @@ def compare_lists(list1, list2): return moins, plus -@record class event(BasicService): """Event service class. It extends BasicService, but should not implement any change trigger, since it's this service which is designed to call @@ -187,5 +191,5 @@ class event(BasicService): # trigger_send(*msg) def trigger_send(routing_key, body): - sender = Event("civet") + sender = EventProducer("civet") sender.send_message("trigger.%s" % (routing_key,), body) diff --git a/gestion/trigger/services/firewall.py b/gestion/trigger/services/firewall.py index 4eda52e9..83ff0981 100644 --- a/gestion/trigger/services/firewall.py +++ b/gestion/trigger/services/firewall.py @@ -15,57 +15,22 @@ it to regenerate what needs to. """ import lc_ldap.shortcuts -from gestion.trigger.host import record +import gestion.config.trigger as trigger_config from gestion.trigger.services.service import BasicService import cranslib.clogger as clogger -import gestion.config.firewall as firewall_config import gestion.trigger.firewall4.firewall4 as firewall4 -logger = clogger.CLogger("trigger.firewall", "debug") +logger = clogger.CLogger("trigger", "firewall", "debug", trigger_config.debug) -class FwFunFactory(object): - """Factory containing which function is part of the trigger set - - """ - - _meths = {} - - @classmethod - def register(cls, key, value): - """Stores in factory the function name and its value - - """ - cls._meths[key] = value - - @classmethod - def get(cls, key): - """Gets what is stored - - """ - return cls._meths.get(key, None) - -def fwrecord(function): - """Records function in FwFunFactory - - """ - FwFunFactory.register(function.func_name, function) - -def fwcall(fwfun): - """Calls in function from FwFunFactory - - """ - return FwFunFactory.get(fwfun) - -@record -class firewall(BasicService): +class Firewall(BasicService): """Firewall service that handles any modification in the firewall. """ # Class lookup table to define which changes call which function. changes_trigger = { - lc_ldap.attributs.macAddress.ldap_name: (firewall.send_mac_ip,), - lc_ldap.attributs.ipHostNumber.ldap_name: (firewall.send_mac_ip,), + lc_ldap.attributs.macAddress.ldap_name: ('send_mac_ip',), + lc_ldap.attributs.ipHostNumber.ldap_name: ('send_mac_ip',), } @classmethod @@ -98,19 +63,19 @@ class firewall(BasicService): return (service, data) = body logger.info("Calling service %s for data %r", service, data) - fwcall(service)(data) + getattr(cls, service)(data) -@fwrecord -def mac_ip(body): - host_fw = firewall4.firewall() - if body and isinstance(body, dict): - for (mac, ip) in body.get("add", []): - logger.info("Adding mac_ip %s,%s", mac, ip) - host_fw.mac_ip_append(mac, ip) - for (mac, ip) in body.get("delete", []): - logger.info("Removing mac_ip %s,%s", mac, ip) - host_fw.mac_ip_remove(mac, ip) - for (rmac, rip, mac, ip) in body.get("update", []): - logger.info("Updating mac_ip %s,%s with %s,%s", rmac, rip, mac, ip) - host_fw.mac_ip_remove(rmac, rip) - host_fw.mac_ip_append(mac, ip) + @classmethod + def mac_ip(cls, body): + host_fw = firewall4.firewall() + if body and isinstance(body, dict): + for (mac, ip) in body.get("add", []): + logger.info("Adding mac_ip %s,%s", mac, ip) + host_fw.mac_ip_append(mac, ip) + for (mac, ip) in body.get("delete", []): + logger.info("Removing mac_ip %s,%s", mac, ip) + host_fw.mac_ip_remove(mac, ip) + for (rmac, rip, mac, ip) in body.get("update", []): + logger.info("Updating mac_ip %s,%s with %s,%s", rmac, rip, mac, ip) + host_fw.mac_ip_remove(rmac, rip) + host_fw.mac_ip_append(mac, ip) diff --git a/gestion/trigger/services/service.py b/gestion/trigger/services/service.py index aa9589d1..42e16407 100644 --- a/gestion/trigger/services/service.py +++ b/gestion/trigger/services/service.py @@ -6,11 +6,65 @@ This module provides a basic service class to other services. It should *NOT* be referenced in configuration of trigger. """ +import collections + +import cranslib.clogger as clogger +import gestion.config.trigger as trigger_config +from gestion.trigger.host import TriggerFactory + +logger = clogger.CLogger("trigger", "service", "debug", trigger_config.debug) + +class MetaService(type): + """Metaclass designed to handle all services. + + """ + + def __new__(mcs, cname, cpar, cattrs): + """Method producing the new class itself + At first, I wanted to put the changes_trigger modification in __new__, + using direct modification of cattrs['changes_trigger'] by pointing the + required methods (classmethods). The problem was that these methods were + bound at the return of type.__new__, for a reason I could not exactly + explain. + + I found a workaround using __init__, so the point would be to remove + __new__, and directly use type.__new__, but this comment seems useful, + so __new__ will survive. + + """ + return super(MetaService, mcs).__new__(mcs, cname, cpar, cattrs) + + def __init__(cls, cname, cpar, cattrs): + """Used to register the generated classes in TriggerFactory, and modify the behavior of + changes_trigger by pointing functions instead of their names. This allows to cancel any + positional requirement in class definition. + + Do NEVER return something in __init__ function. + + """ + if not cname == "BasicService": + TriggerFactory.register(cname.lower(), cls) + changes_trigger = collections.defaultdict(list) + # I love getattr + text_changes_trigger = getattr(cls, "changes_trigger", {}) + for (ldap_attr_name, funcs_name) in text_changes_trigger.items(): + for func_name in funcs_name: + # I really love getattr. + get = getattr(cls, func_name, None) + if get is None: + logger.critical("Fatal, bad function (%r) reference in %r.", func_name, cname) + continue + changes_trigger[ldap_attr_name].append(get) + setattr(cls, "changes_trigger", changes_trigger) + super(MetaService, cls).__init__(cname, cpar, cattrs) + class BasicService(object): """Basic service handler. Other services should inherit fron this one. """ + __metaclass__ = MetaService + changes_trigger = {} @classmethod diff --git a/gestion/trigger/trigger.py b/gestion/trigger/trigger.py index 9a55c34e..b3bf939f 100755 --- a/gestion/trigger/trigger.py +++ b/gestion/trigger/trigger.py @@ -11,16 +11,19 @@ # Date : 29/04/2014 import argparse -import cranslib.clogger as clogger -import cmb import cPickle import socket +import traceback +import sys + import gestion.config.trigger as trigger_config import gestion.affichage as affichage -import sys from gestion.trigger.host import trigger +import cranslib.clogger as clogger +import cmb hostname = socket.gethostname().split(".")[0] +logger = clogger.CLogger("trigger", "trigger", "info", trigger_config.debug) # Ce bloc contient le peu de "magie" de la librairie, on utilise les services listés dans config/trigger.py # comme référence. Pour éviter toute redondance, la commande importe donc les services utiles suivant cette @@ -30,8 +33,10 @@ hostname = socket.gethostname().split(".")[0] import importlib services = {} for config_service in trigger_config.services[hostname]: - services[config_service] = importlib.import_module("gestion.trigger.services.%s" % (config_service,)) -logger = clogger.CLogger("trigger", "info") + try: + services[config_service] = importlib.import_module("gestion.trigger.services.%s" % (config_service,)) + except Exception as e: + logger.critical("Fatal : import of %r failed, see following traceback. %r", config_service, traceback.format_exc()) class EvenementListener(cmb.AsynchronousConsumer): """