diff --git a/bcfg2new/Plugins/Python/PythonDefaults.py b/bcfg2new/Plugins/Python/PythonDefaults.py new file mode 100644 index 00000000..66846799 --- /dev/null +++ b/bcfg2new/Plugins/Python/PythonDefaults.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Contient les valeurs par défaut du plugin python +de Bcfg2""" + +DEFAULT_USER = 'root' +DEFAULT_GROUP = 'root' +DEFAULT_ACLS = 0644 + +INCLUDES = "../etc/python" diff --git a/bcfg2new/Plugins/Python/PythonEnv.py b/bcfg2new/Plugins/Python/PythonEnv.py new file mode 100644 index 00000000..bf2274dc --- /dev/null +++ b/bcfg2new/Plugins/Python/PythonEnv.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python2.7 +# -*- coding: utf-8 -*- +"""SafeEnvironment implementation for use of exec""" + +import os +import cStringIO + +import PythonDefaults +import PythonFile + +class SafeEnvironment(dict): + """Environnement isolé dans lequel on exécute un script""" + + def __init__(self, additionnal=None, parent=None): + # Création de l'environment initial + super(self.__class__, self).__init__({ + # Écrit: variable keysep tostring(value) + "defvar": self.defvar, + # La convertion en chaîne de charactère + "tostring": self.tostring, + # Définition des convertions + "conv": { + bool: { + True: "yes", + False: "no", + }, + list: lambda l: ", ".join([ + str(x) + for x in l + ]), + tuple: lambda l: ", ".join([ + str(x) + for x in l + ]), + }, + # Fonction de base pour imprimer quelque chose + "out": self.out, + # Le séparateur pour la forme: variable keysep valeur + "keysep": "=", + # Le charactère de commentaire + "comment_start": "#", + # Du mapping de certaines fonctions + "include": self.include, + # Du mapping de certaines fonctions + "dump": self.dump, + # Infos standard pour le fichier (écrasable localement) + "info": { + 'owner': PythonDefaults.DEFAULT_USER, + 'group': PythonDefaults.DEFAULT_GROUP, + 'mode': PythonDefaults.DEFAULT_ACLS, + } + }) + + if additionnal is None: + additionnal = {} + super(self.__class__, self).update(additionnal) + + # On crée le flux dans lequel le fichier de config sera généré + self.stream = cStringIO.StringIO() + + # Le Pythonfile parent est référencé ici + self.parent = parent + + # Les trucs inclus + self.included = [] + + def __setitem__(self, variable, value): + """Lorsqu'on définit une variable, si elle est listée dans la variable + exports, on l'incorpore dans le fichier produit""" + super(self.__class__, self).__setitem__(variable, value) + + def defvar(self, variable, value): + """Quand on fait un export, on utilise defvar pour incorporer la variable + et sa valeur dans le fichier produit""" + # On écrit mavariable = toto, en appliquant une éventuelle conversion à toto + self.out("%s%s%s\n" % (variable, self['keysep'], self.tostring(value))) + + def out(self, string): + """C'est le print local. Sauf qu'on écrit dans self.stream. C'est pas + indispensable, car l'évaluation du fichier se fait déjà à sys.stdout + pointant vers ledit stream.""" + self.stream.write(string) + + def tostring(self, value): + """On convertit un objet python dans un format "string" sympa. + En vrai c'est horrible et il faudrait virer ce genre de kludge.""" + convertor = self["conv"].get(type(value)) + if convertor: + if type(convertor) == dict: + return convertor[value] + else: + return convertor(value) + else: + return str(value) + + def dump(self, incfile): + """On exécute le fichier python dans l'environnement courant + + incfile est le nom du fichier, sans le .py""" + filename = os.path.join(self.parent.parent.include, "%s.py" % (incfile,)) + python_file = PythonFile.PythonFile(filename, self.parent.parent) + python_file.run(environment=self) + + def include(self, incfile): + """Pareil qu'au dessus, mais on ne le fait que si ça n'a pas + été fait""" + if incfile in self.included: + return + self.included.append(incfile) + self.dump(incfile) diff --git a/bcfg2new/Plugins/Python/PythonFactories.py b/bcfg2new/Plugins/Python/PythonFactories.py new file mode 100644 index 00000000..f8fb63b3 --- /dev/null +++ b/bcfg2new/Plugins/Python/PythonFactories.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python2.7 +# -*- coding: utf-8 -*- +"""Ce module est prévu pour héberger des factories, stockant toute +instance d'un fichier Python déjà compilé.""" + +class PythonFileFactory(object): + """Cette Factory stocke l'ensemble des fichiers Python déjà instanciés. + Elle garantit entre autre leur unicité dans le fonctionnement du plugin""" + + #: Stocke la liste des instances avec leur chemin absolu. + files = {} + + @classmethod + def get(cls, path): + """Récupère l'instance si elle existe, ou renvoit None""" + return cls.files.get(path, None) + + @classmethod + def record(cls, path, instance): + """Enregistre l'instance dans la Factory""" + cls.files[path] = instance + + @classmethod + def flush_one(cls, path): + """Vire une instance du dico""" + instance_to_delete = cls.files.pop(path, None) + del instance_to_delete + + @classmethod + def flush(cls): + """Vire toutes les instances du dico""" + for path in cls.files.keys(): + cls.flush_one(path) diff --git a/bcfg2new/Plugins/Python/PythonFile.py b/bcfg2new/Plugins/Python/PythonFile.py new file mode 100644 index 00000000..68da571e --- /dev/null +++ b/bcfg2new/Plugins/Python/PythonFile.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python2.7 +# -*- coding: utf-8 -*- +"""Fournit une couche d'abstraction Python pour les fichiers du même +nom""" + +import os +import sys +import re +import marshal +import cStringIO + +from Bcfg2.Server.Plugin import Debuggable + +from .PythonFactories import PythonFileFactory +import PythonEnv +import PythonTools + +__RE_SPECIAL_LINE = re.compile(r"^([ \t]*)(@|%)(.*)$", re.MULTILINE) +__RE_AFFECTATION = re.compile(r"([a-zA-Z_][a-zA-Z_0-9]*)[ \t]*=") +__RE_SPACE_SEP = re.compile(r"([^ \t]*)[ \t]+=?(.*)") + +class PythonFile(Debuggable): + """Classe représentant un fichier Python""" + + #: Permet de savoir si l'instance a déjà été initialisée + initialized = False + + def __new__(cls, path, parent=None): + """Si le fichier a déjà été enregistré dans la Factory, on + le retourne, et on évite de réinstancier la classe. + + path est le chemin absolu du fichier""" + + path = os.path.normpath(path) + + file_instance = PythonFileFactory.get(path) + if file_instance is None: + file_instance = super(PythonFile, cls).__new__(cls) + PythonFileFactory.record(path, file_instance) + + return file_instance + + def __init__(self, path, parent=None): + """Initialisation, si non déjà faite""" + + if self.initialized: + return + + super(self.__class__, self).__init__() + + #: A string containing the raw data in this file + self.data = None + + #: Le chemin complet du fichier + self.path = os.path.normpath(path) + + #: Le nom du fichier + self.name = os.path.basename(self.path) + + #: Un logger + self.logger = PythonTools.LOGGER + + #: Le plugin parent est pointé pour des raisons pratiques + self.parent = parent + + #: C'est bon, c'est initialisé + self.initialized = True + + def exists(self): + """Teste l'existence du fichier""" + return os.path.exists(self.path) + + def HandleEvent(self, event=None): + """ HandleEvent is called whenever the FAM registers an event. + + :param event: The event object + :type event: Bcfg2.Server.FileMonitor.Event + :returns: None + """ + if event and event.code2str() not in ['exists', 'changed', 'created']: + return + + try: + self.load() + except IOError: + err = sys.exc_info()[1] + self.logger.error("Failed to read file %s: %s" % (self.name, err)) + except: + err = sys.exc_info()[1] + self.logger.error("Failed to parse file %s: %s" % (self.name, err)) + + def __repr__(self): + return "%s: %s" % (self.__class__.__name__, self.name) + + def load(self, refresh=True): + """Charge le fichier""" + if self.data is not None and not refresh: + return + + try: + directory = os.path.dirname(self.path) + compiled_file = os.path.join(directory, ".%s.COMPILED" % (self.name,)) + + if os.path.exists(compiled_file) and os.stat(self.path).st_mtime <= os.stat(compiled_file).st_mtime: + self.data = marshal.load(open(compiled_file, 'r')) + else: + self.data = compileSource(open(self.path, 'r').read(), self.path, self.logger) + cfile = open(compiled_file, "w") + marshal.dump(self.data, cfile) + cfile.close() + except Exception as error: + PythonTools.log_traceback(self.path, 'compilation', error, self.logger) + + def run(self, additionnal=None, environment=None): + """Exécute le code""" + if self.data is None: + self.load(True) + + if additionnal is None: + additionnal = {} + + if environment is None: + environment = PythonEnv.SafeEnvironment(additionnal, self) + + # On ne réaffecte stdout que sur le premier élément de la pile + have_to_get_out = False + + if sys.stdout == sys.__stdout__: + sys.stdout = environment.stream + have_to_get_out = True + + # Lors de l'exécution d'un fichier, on inclut + # toujours common (ie on l'exécute dans l'environnement) + environment.include("common") + + try: + exec(self.data, environment) + except Exception: + sys.stderr.write('code: %r\n' % (self.data,)) + sys.stdout = sys.__stdout__ + raise + + if sys.stdout != sys.__stdout__ and have_to_get_out: + sys.stdout = sys.__stdout__ + have_to_get_out = False + return environment.stream.getvalue(), environment['info'] + +#+---------------------------------------------+ +#| Tools for compilation | +#+---------------------------------------------+ + +def compileSource(source, filename="", logger=None): + '''Compile un script''' + # On commence par remplacer les lignes de la forme + # @xxx par out("xxx") + newsource = cStringIO.StringIO() + start = 0 + + # Parsing de goret : on boucle sur les lignes spéciales, + # c'est-à-dire celles commençant par un @ ou un % précédé + # par d'éventuelles espaces/tabs. + for match in __RE_SPECIAL_LINE.finditer(source): + # On prend tout ce qui ne nous intéresse pas et on l'ajoute. + newsource.write(source[start:match.start()]) + + # On redéfinit start. + start = match.end() + + # On écrit le premier groupe (les espaces et cie) + newsource.write(match.group(1)) + + # Le linetype est soit @ soit % + linetype = match.group(2) + + # @ c'est du print. + if linetype == "@": + # On prend ce qui nous intéresse, et on fait quelques remplacements + # pour éviter les plantages. + line = match.group(3).replace("\\", "\\\\").replace('"', '\\"') + + # Si la ligne est un commentaire, on la reproduit en remplaçant éventuellement + # le # par le bon caractère. + if line and line[0] == "#": + newsource.write('out(comment_start + "') + line = line[1:] + + # Sinon bah.... + else: + newsource.write('out("') + + # On écrit ladite ligne + newsource.write(line) + + # Et un superbe \n. + newsource.write('\\n")') + + # %, affectation. + elif linetype == "%": + # On récupère le reste. + line = match.group(3) + + # On fait du matching clef/valeur + match = __RE_AFFECTATION.match(line) + if match: + # Le nom est le premier groupe. + # Et après c'est weird... + varname = match.group(1) + newsource.write(line) + newsource.write("; defvar('") + newsource.write(varname) + newsource.write("', tostring(") + newsource.write(varname) + newsource.write("))\n") + else: + # Pareil, sauf que cette fois, ce qu'on fait a un sens. + match = __RE_SPACE_SEP.match(line) + newsource.write("defvar('") + newsource.write(match.group(1)) + # Le tostring est facultatif. + newsource.write("', tostring(") + newsource.write(match.group(2)) + newsource.write("))\n") + # On continue. + newsource.write(source[start:]) + if logger: + try: + logger.info(newsource.getvalue()) + except: + print "Le logger de BCFG2 c'est de la merde, il refuse le non ascii." + print "Voici ce que j'ai essayé de logguer." + print newsource.getvalue() + return compile(newsource.getvalue(), filename, "exec") diff --git a/bcfg2new/Plugins/Python/PythonPlugin.py b/bcfg2new/Plugins/Python/PythonPlugin.py new file mode 100644 index 00000000..2a6c6759 --- /dev/null +++ b/bcfg2new/Plugins/Python/PythonPlugin.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# PythonPlugin.py +# --------- +# +# Copyright © 2015 Pierre-Elliott Bécue +"""Plugin servant à gérer des fichiers python, dont la sortie sera +la configuration d'un client.""" + +#: N'exporte que la classe Python +__all__ = [ + "Python", +] + +import os +import re +import binascii + +from Bcfg2.Server.Plugin import Plugin, Generator, PluginExecutionError, track_statistics +from Bcfg2.Server.Plugin.base import Debuggable + +import PythonTools +import PythonDefaults +import PythonFile + +class Python(Plugin, Generator, Debuggable): + """Générateur offrant des fonctionnalités de templating pour les fichiers python""" + + name = 'Python' + + #: Les DirectoryBacked ont des fonctions de monitoring + #: intégrées. Quand des changements arrivent sur les dossiers, + #: c'est la merde, il est préférable de relancer Bcfg2, car + #: FileMonitor ne sait pas démonitorer/remonitorer. + #: En revanche, pour les fichiers, il appelle __child__ comme + #: "générateur" pour les trucs à surveiller. Quand un fichier + #: est créé/modifié, sa méthode HandleEvent est appelée. + __child__ = PythonFile.PythonFile + + #: Ce module gère plein de choses. + patterns = re.compile(r'.*') + + #: Ignore ces chemins spécifiques + ignore = re.compile(r'.*\.(COMPILED|pyc)') + + __version__ = '2.0' + + __author__ = 'becue@crans.org' + + def __init__(self, core, datastore): + """Pour initialiser le plugin""" + + #: Initialise un certain nombre de choses en background + Plugin.__init__(self, core, datastore) + Debuggable.__init__(self) + + #: self.entries contains information about the files monitored + #: by this object. The keys of the dict are the relative + #: paths to the files. The values are the objects (of type + #: :attr:`__child__`) that handle their contents. + self.entries = {} + + #: self.handles contains information about the directories + #: monitored by this object. The keys of the dict are the + #: values returned by the initial fam.AddMonitor() call (which + #: appear to be integers). The values are the relative paths of + #: the directories. + self.handles = {} + + #: FileMonitor + self.fam = self.core.fam + + #: Monitor everything in the plugin's directory + if not os.path.exists(self.data): + self.logger.warning("%s does not exist, creating" % (self.data,)) + os.makedirs(self.data) + self.add_directory_monitor('') + + #: Dossier des includes + self.include = os.path.abspath(os.path.join(self.data, PythonDefaults.INCLUDES)) + + #: Quand on initialise un DirectoryBacked, on a déjà un monitoring de + #: self.data, donc on a besoin que des includes + self.add_directory_monitor(PythonDefaults.INCLUDES) + + @track_statistics() + def HandlesEntry(self, entry, metadata): + """Vérifie si l'entrée est gérée par le plugin""" + relpath = entry.get('name')[1:] + if relpath in self.entries: + return True + return False + + @track_statistics() + def HandleEntry(self, entry, metadata): + """Construit le fichier demandé""" + # On récupère le code qui va bien. + relpath = entry.get('name')[1:] + python_file = self.entries[relpath] + + # Et le nom de fichier. + fname = entry.get('realname', entry.get('name')) + + # Si on est en débug, on loggue ce qu'on fait. + PythonTools.debug("Building config file: %s" % (fname,), PythonTools.LOGGER, 'blue') + + # On crée un environnement autonome pour exécuter le fichier. + additionnal = { + 'metadata': metadata, + } + + try: + text, info = python_file.run(additionnal) + except Exception as error: + PythonTools.log_traceback(fname, 'exec', error, PythonTools.LOGGER) + raise PluginExecutionError + + # On récupère les infos + if info.get('encoding', '') == 'base64': + text = binascii.b2a_base64(text) + + # lxml n'accepte que de l'ascii ou de l'unicode + # donc faut décoder. + try: + entry.text = text.decode("UTF-8") + except: + # solution de fallback + entry.text = text.decode("ISO-8859-15") + + # En cas de débug, on stocke les données + PythonTools.debug(entry.text, PythonTools.LOGGER) + + # On récupère les permissions depuis le dico "info". + # En théorie, les valeurs par défaut ne devraient pas être utilisées + # Car elles sont déjà affectées dans Pygen + entry.attrib['owner'] = info.get('owner', PythonDefaults.DEFAULT_USER) + entry.attrib['group'] = info.get('group', PythonDefaults.DEFAULT_GROUP) + entry.attrib['mode'] = oct(info.get('mode', PythonDefaults.DEFAULT_ACLS)) + + if 'encoding' in info: + entry.attrib['encoding'] = info['encoding'] + + def add_directory_monitor(self, relative): + """ Add a new directory to the FAM for monitoring. + + :param relative: Path name to monitor. This must be relative + to the plugin's directory. An empty string + value ("") will cause the plugin directory + itself to be monitored. + :type relative: string + :returns: None + """ + + #: On normalise pour éviter des problèmes quand le FileMonitor + #: voit des changements par la suite. + #: Les chemins sont absolus pour la même raison. + dirpathname = os.path.normpath(os.path.join(self.data, relative)) + if relative not in self.handles.values(): + if not os.path.isdir(dirpathname): + self.logger.error("%s is not a directory" % (dirpathname,)) + return + #: reqid est un chemin absolu sans trailing slash + reqid = self.fam.AddMonitor(dirpathname, self) + self.handles[reqid] = relative + + def add_entry(self, relative, event): + """ Add a new file to our tracked entries, and to our FAM for + monitoring. + + :param relative: Path name to monitor. This must be relative + to the plugin's directory. + :type relative: string: + :param event: FAM event that caused this entry to be added. + :type event: Bcfg2.Server.FileMonitor.Event + :returns: None + """ + #: Les entrées sont en relatif depuis le dossier de config + self.entries[relative] = self.__child__( + os.path.join(self.data, relative), + self + ) + self.entries[relative].HandleEvent(event) + + def HandleEvent(self, event): + """ Handle FAM events. + + This method is invoked by the FAM when it detects a change to + a filesystem object we have requsted to be monitored. + + This method manages the lifecycle of events related to the + monitored objects, adding them to our list of entries and + creating objects of type :attr:`__child__` that actually do + the domain-specific processing. When appropriate, it + propogates events those objects by invoking their HandleEvent + method in turn. + + :param event: FAM event that caused this entry to be added. + :type event: Bcfg2.Server.FileMonitor.Event + :returns: None + """ + action = event.code2str() + + # Exclude events for actions we don't care about + if action == 'endExist': + return + + if event.requestID not in self.handles: + self.logger.warn( + "Got %s event with unknown handle (%s) for %s" % (action, event.requestID, event.filename) + ) + return + + # Clean up path names + event.filename = os.path.normpath(event.filename) + if event.filename.startswith(self.data) or os.path.normpath(event.requestID) == event.filename: + # the first event we get is on the data directory itself + event.filename = event.filename[len(os.path.normpath(event.requestID)) + 1:] + + if self.ignore and self.ignore.search(event.filename): + self.logger.debug("Ignoring event %s" % (event.filename,)) + return + + # Calculate the absolute and relative paths this event refers to + abspath = os.path.join(self.data, self.handles[event.requestID], + event.filename) + relpath = os.path.join(self.handles[event.requestID], + event.filename).lstrip('/') + + if action == 'deleted': + for key in list(self.entries.keys()): + if key.startswith(relpath): + del self.entries[key] + # We remove values from self.entries, but not + # self.handles, because the FileMonitor doesn't stop + # watching a directory just because it gets deleted. If it + # is recreated, we will start getting notifications for it + # again without having to add a new monitor. + elif os.path.isdir(abspath): + # Deal with events for directories + if action in ['exists', 'created']: + self.add_directory_monitor(relpath) + elif action == 'changed': + if relpath in self.entries: + # Ownerships, permissions or timestamps changed on + # the directory. None of these should affect the + # contents of the files, though it could change + # our ability to access them. + # + # It seems like the right thing to do is to cancel + # monitoring the directory and then begin + # monitoring it again. But the current FileMonitor + # class doesn't support canceling, so at least let + # the user know that a restart might be a good + # idea. + self.logger.warn( + "Directory properties for %s changed, please consider restarting the server" % (abspath) + ) + else: + # Got a "changed" event for a directory that we + # didn't know about. Go ahead and treat it like a + # "created" event, but log a warning, because this + # is unexpected. + self.logger.warn( + "Got %s event for unexpected dir %s" % (action, abspath) + ) + self.add_directory_monitor(relpath) + else: + self.logger.warn( + "Got unknown dir event %s %s %s" % (event.requestID, event.code2str(), abspath) + ) + elif self.patterns.search(event.filename): + if action in ['exists', 'created']: + self.add_entry(relpath, event) + elif action == 'changed': + if relpath in self.entries: + self.entries[relpath].HandleEvent(event) + else: + # Got a "changed" event for a file that we didn't + # know about. Go ahead and treat it like a + # "created" event, but log a warning, because this + # is unexpected. + self.logger.warn( + "Got %s event for unexpected file %s" % (action, abspath) + ) + self.add_entry(relpath, event) + else: + self.logger.warn( + "Got unknown file event %s %s %s" % (event.requestID, event.code2str(), abspath) + ) + else: + self.logger.warn( + "Could not process filename %s; ignoring" % (event.filename) + ) diff --git a/bcfg2new/Plugins/Python/PythonTools.py b/bcfg2new/Plugins/Python/PythonTools.py new file mode 100644 index 00000000..41c859b7 --- /dev/null +++ b/bcfg2new/Plugins/Python/PythonTools.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Fournit quelques outils pour le plugin Python""" + +import os +import logging +import cStringIO +import traceback + +LOGGER = logging.getLogger('Bcfg2.Plugins.Python') + +COLOR_CODE = { + 'grey': 30, + 'red': 31, + 'green': 32, + 'yellow': 33, + 'blue': 34, + 'purple': 35, + 'cyan': 36, +} + +BCFG2_DEBUG = os.getenv("BCFG2_DEBUG") +BCFG2_DEBUG_COLOR = os.getenv("BCFG2_DEBUG_COLOR") + +def debug(message, logger, color=None): + """Stocke dans un logger les messages de debug""" + if not BCFG2_DEBUG: + return + + if BCFG2_DEBUG_COLOR and color: + logger.info("\033[1;%dm%s\033[0m" % (COLOR_CODE[color], message)) + else: + logger.info(message) + +def log_traceback(fname, section, exn, logger): + """En cas de traceback, on le loggue sans faire planter + le serveur bcfg2""" + logger.error('Python %s error: %s: %s: %s' % (section, fname, str(exn.__class__).split('.', 2)[1], str(exn))) + + stream = cStringIO.StringIO() + traceback.print_exc(file=stream) + + for line in stream.getvalue().splitlines(): + logger.error('Python %s error: -> %s' % (section, line)) + +class PythonIncludePaths(object): + """C'est un objet qui stocke les dossier d'inclusion python""" + includes = [] + + @classmethod + def get(cls, index, default): + """Retourne includes[index] ou default""" + if len(cls.includes) > index: + return cls.includes[index] + return default + + @classmethod + def append(cls, value): + """Ajoute une valeur à la liste""" + cls.includes.append(value) + + @classmethod + def remove(cls, value): + """Retire une valeur à la liste""" + if value in cls.includes: + cls.includes.remove(value) + + @classmethod + def pop(cls, index): + """Vire un index si existant""" + if len(cls.includes) > index: + return cls.includes.pop(index) + diff --git a/bcfg2new/Plugins/Python/__init__.py b/bcfg2new/Plugins/Python/__init__.py new file mode 100644 index 00000000..cb4f2756 --- /dev/null +++ b/bcfg2new/Plugins/Python/__init__.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python2.7 +# -*- coding: utf-8 -*- +"""Python plugin initializator for +Bcfg2""" + +from .PythonPlugin import Python diff --git a/bcfg2new/Tools/Python.py b/bcfg2new/Tools/Python.py new file mode 100644 index 00000000..06fe2f54 --- /dev/null +++ b/bcfg2new/Tools/Python.py @@ -0,0 +1,683 @@ +"""All Python Type client support for Bcfg2.""" +__revision__ = '$Revision$' + +import binascii +from datetime import datetime +import difflib +import errno +import grp +import logging +import os +import pwd +import shutil +import stat +import sys +import time +# py3k compatibility +if sys.hexversion >= 0x03000000: + unicode = str + +import Bcfg2.Client.Tools +import Bcfg2.Options +from Bcfg2.Client import XML + +log = logging.getLogger('python') + +# map between dev_type attribute and stat constants +device_map = {'block': stat.S_IFBLK, + 'char': stat.S_IFCHR, + 'fifo': stat.S_IFIFO} + + +def calcMode(initial, mode): + """This compares ondisk permissions with specified ones.""" + pdisp = [{1:stat.S_ISVTX, 2:stat.S_ISGID, 4:stat.S_ISUID}, + {1:stat.S_IXUSR, 2:stat.S_IWUSR, 4:stat.S_IRUSR}, + {1:stat.S_IXGRP, 2:stat.S_IWGRP, 4:stat.S_IRGRP}, + {1:stat.S_IXOTH, 2:stat.S_IWOTH, 4:stat.S_IROTH}] + tempmode = initial + if len(mode) == 3: + mode = '0%s' % (mode) + pdigits = [int(mode[digit]) for digit in range(4)] + for index in range(4): + for (num, perm) in list(pdisp[index].items()): + if pdigits[index] & num: + tempmode |= perm + return tempmode + + +def normGid(entry): + """ + This takes a group name or gid and + returns the corresponding gid or False. + """ + try: + try: + return int(entry.get('group')) + except: + return int(grp.getgrnam(entry.get('group'))[2]) + except (OSError, KeyError): + log.error('GID normalization failed for %s. Does group %s exist?' + % (entry.get('name'), entry.get('group'))) + return False + + +def normUid(entry): + """ + This takes a user name or uid and + returns the corresponding uid or False. + """ + try: + try: + return int(entry.get('owner')) + except: + return int(pwd.getpwnam(entry.get('owner'))[2]) + except (OSError, KeyError): + log.error('UID normalization failed for %s. Does owner %s exist?' + % (entry.get('name'), entry.get('owner'))) + return False + + +def isString(strng, encoding): + """ + Returns true if the string contains no ASCII control characters + and can be decoded from the specified encoding. + """ + for char in strng: + if ord(char) < 9 or ord(char) > 13 and ord(char) < 32: + return False + try: + strng.decode(encoding) + return True + except: + return False + + +class Python(Bcfg2.Client.Tools.Tool): + """Python File support code.""" + name = 'Python' + __handles__ = [('Python', 'file'), + ('Python', None)] + __req__ = {'Python': ['name']} + + # grab paranoid options from /etc/bcfg2.conf + opts = {'ppath': Bcfg2.Options.PARANOID_PATH, + 'max_copies': Bcfg2.Options.PARANOID_MAX_COPIES} + setup = Bcfg2.Options.OptionParser(opts) + setup.parse([]) + ppath = setup['ppath'] + max_copies = setup['max_copies'] + + def canInstall(self, entry): + """Check if entry is complete for installation.""" + if Bcfg2.Client.Tools.Tool.canInstall(self, entry): + if (entry.tag, + entry.get('type'), + entry.text, + entry.get('empty', 'false')) == ('Python', + 'file', + None, + 'false'): + return False + return True + else: + return False + + def gatherCurrentData(self, entry): + if entry.tag == 'Python' and entry.get('type') == 'file': + try: + ondisk = os.stat(entry.get('name')) + except OSError: + entry.set('current_exists', 'false') + self.logger.debug("%s %s does not exist" % + (entry.tag, entry.get('name'))) + return False + try: + entry.set('current_owner', str(ondisk[stat.ST_UID])) + entry.set('current_group', str(ondisk[stat.ST_GID])) + except (OSError, KeyError): + pass + entry.set('mode', str(oct(ondisk[stat.ST_MODE])[-4:])) + + def Verifydirectory(self, entry, modlist): + """Verify Path type='directory' entry.""" + if entry.get('mode') == None or \ + entry.get('owner') == None or \ + entry.get('group') == None: + self.logger.error('Entry %s not completely specified. ' + 'Try running bcfg2-lint.' % (entry.get('name'))) + return False + while len(entry.get('mode', '')) < 4: + entry.set('mode', '0' + entry.get('mode', '')) + try: + ondisk = os.stat(entry.get('name')) + except OSError: + entry.set('current_exists', 'false') + self.logger.debug("%s %s does not exist" % + (entry.tag, entry.get('name'))) + return False + try: + owner = str(ondisk[stat.ST_UID]) + group = str(ondisk[stat.ST_GID]) + except (OSError, KeyError): + self.logger.error('User/Group resolution failed for path %s' % \ + entry.get('name')) + owner = 'root' + group = '0' + finfo = os.stat(entry.get('name')) + mode = oct(finfo[stat.ST_MODE])[-4:] + if entry.get('mtime', '-1') != '-1': + mtime = str(finfo[stat.ST_MTIME]) + else: + mtime = '-1' + pTrue = ((owner == str(normUid(entry))) and + (group == str(normGid(entry))) and + (mode == entry.get('mode')) and + (mtime == entry.get('mtime', '-1'))) + + pruneTrue = True + ex_ents = [] + if entry.get('prune', 'false') == 'true' \ + and (entry.tag == 'Path' and entry.get('type') == 'directory'): + # check for any extra entries when prune='true' attribute is set + try: + entries = ['/'.join([entry.get('name'), ent]) \ + for ent in os.listdir(entry.get('name'))] + ex_ents = [e for e in entries if e not in modlist] + if ex_ents: + pruneTrue = False + self.logger.debug("Directory %s contains extra entries:" % \ + entry.get('name')) + self.logger.debug(ex_ents) + nqtext = entry.get('qtext', '') + '\n' + nqtext += "Directory %s contains extra entries:" % \ + entry.get('name') + nqtext += ":".join(ex_ents) + entry.set('qtest', nqtext) + [entry.append(XML.Element('Prune', path=x)) \ + for x in ex_ents] + except OSError: + ex_ents = [] + pruneTrue = True + + if not pTrue: + if owner != str(normUid(entry)): + entry.set('current_owner', owner) + self.logger.debug("%s %s ownership wrong" % \ + (entry.tag, entry.get('name'))) + nqtext = entry.get('qtext', '') + '\n' + nqtext += "%s owner wrong. is %s should be %s" % \ + (entry.get('name'), owner, entry.get('owner')) + entry.set('qtext', nqtext) + if group != str(normGid(entry)): + entry.set('current_group', group) + self.logger.debug("%s %s group wrong" % \ + (entry.tag, entry.get('name'))) + nqtext = entry.get('qtext', '') + '\n' + nqtext += "%s group is %s should be %s" % \ + (entry.get('name'), group, entry.get('group')) + entry.set('qtext', nqtext) + if mode != entry.get('mode'): + entry.set('current_mode', mode) + self.logger.debug("%s %s permissions are %s should be %s" % + (entry.tag, + entry.get('name'), + mode, + entry.get('mode'))) + nqtext = entry.get('qtext', '') + '\n' + nqtext += "%s %s mode are %s should be %s" % \ + (entry.tag, + entry.get('name'), + mode, + entry.get('mode')) + entry.set('qtext', nqtext) + if mtime != entry.get('mtime', '-1'): + entry.set('current_mtime', mtime) + self.logger.debug("%s %s mtime is %s should be %s" \ + % (entry.tag, entry.get('name'), mtime, + entry.get('mtime'))) + nqtext = entry.get('qtext', '') + '\n' + nqtext += "%s mtime is %s should be %s" % \ + (entry.get('name'), mtime, entry.get('mtime')) + entry.set('qtext', nqtext) + if entry.get('type') != 'file': + nnqtext = entry.get('qtext') + nnqtext += '\nInstall %s %s: (y/N) ' % (entry.get('type'), + entry.get('name')) + entry.set('qtext', nnqtext) + return pTrue and pruneTrue + + def Installdirectory(self, entry): + """Install Path type='directory' entry.""" + if entry.get('mode') == None or \ + entry.get('owner') == None or \ + entry.get('group') == None: + self.logger.error('Entry %s not completely specified. ' + 'Try running bcfg2-lint.' % \ + (entry.get('name'))) + return False + self.logger.info("Installing directory %s" % (entry.get('name'))) + try: + fmode = os.lstat(entry.get('name')) + if not stat.S_ISDIR(fmode[stat.ST_MODE]): + self.logger.debug("Found a non-directory entry at %s" % \ + (entry.get('name'))) + try: + os.unlink(entry.get('name')) + exists = False + except OSError: + self.logger.info("Failed to unlink %s" % \ + (entry.get('name'))) + return False + else: + self.logger.debug("Found a pre-existing directory at %s" % \ + (entry.get('name'))) + exists = True + except OSError: + # stat failed + exists = False + + if not exists: + parent = "/".join(entry.get('name').split('/')[:-1]) + if parent: + try: + os.stat(parent) + except: + self.logger.debug('Creating parent path for directory %s' % (entry.get('name'))) + for idx in range(len(parent.split('/')[:-1])): + current = '/'+'/'.join(parent.split('/')[1:2+idx]) + try: + sloc = os.stat(current) + except OSError: + try: + os.mkdir(current) + continue + except OSError: + return False + if not stat.S_ISDIR(sloc[stat.ST_MODE]): + try: + os.unlink(current) + os.mkdir(current) + except OSError: + return False + + try: + os.mkdir(entry.get('name')) + except OSError: + self.logger.error('Failed to create directory %s' % \ + (entry.get('name'))) + return False + if entry.get('prune', 'false') == 'true' and entry.get("qtest"): + for pent in entry.findall('Prune'): + pname = pent.get('path') + ulfailed = False + if os.path.isdir(pname): + self.logger.info("Not removing extra directory %s, " + "please check and remove manually" % pname) + continue + try: + self.logger.debug("Unlinking file %s" % pname) + os.unlink(pname) + except OSError: + self.logger.error("Failed to unlink path %s" % pname) + ulfailed = True + if ulfailed: + return False + return self.Installpermissions(entry) + + def Verifyfile(self, entry, _): + """Verify Python type='file' entry.""" + # permissions check + content check + permissionStatus = self.Verifydirectory(entry, _) + tbin = False + if entry.text == None and entry.get('empty', 'false') == 'false': + self.logger.error("Cannot verify incomplete Python type='%s' %s" % + (entry.get('type'), entry.get('name'))) + return False + if entry.get('encoding', 'ascii') == 'base64': + tempdata = binascii.a2b_base64(entry.text) + tbin = True + elif entry.get('empty', 'false') == 'true': + tempdata = '' + else: + tempdata = entry.text + if type(tempdata) == unicode: + try: + tempdata = tempdata.encode(self.setup['encoding']) + except UnicodeEncodeError: + e = sys.exc_info()[1] + self.logger.error("Error encoding file %s:\n %s" % \ + (entry.get('name'), e)) + + different = False + content = None + if not os.path.exists(entry.get("name")): + # first, see if the target file exists at all; if not, + # they're clearly different + different = True + content = "" + else: + # next, see if the size of the target file is different + # from the size of the desired content + try: + estat = os.stat(entry.get('name')) + except OSError: + err = sys.exc_info()[1] + self.logger.error("Failed to stat %s: %s" % + (err.filename, err)) + return False + if len(tempdata) != estat[stat.ST_SIZE]: + different = True + else: + # finally, read in the target file and compare them + # directly. comparison could be done with a checksum, + # which might be faster for big binary files, but + # slower for everything else + try: + content = open(entry.get('name')).read() + except IOError: + err = sys.exc_info()[1] + self.logger.error("Failed to read %s: %s" % + (err.filename, err)) + return False + different = content != tempdata + + if different: + if self.setup['interactive']: + prompt = [entry.get('qtext', '')] + if not tbin and content is None: + # it's possible that we figured out the files are + # different without reading in the local file. if + # the supplied version of the file is not binary, + # we now have to read in the local file to figure + # out if _it_ is binary, and either include that + # fact or the diff in our prompts for -I + try: + content = open(entry.get('name')).read() + except IOError: + err = sys.exc_info()[1] + self.logger.error("Failed to read %s: %s" % + (err.filename, err)) + return False + if tbin or not isString(content, self.setup['encoding']): + # don't compute diffs if the file is binary + prompt.append('Binary file, no printable diff') + else: + diff = self._diff(content, tempdata, + difflib.unified_diff, + filename=entry.get("name")) + if diff: + udiff = '\n'.join(diff) + try: + prompt.append(udiff.decode(self.setup['encoding'])) + except UnicodeDecodeError: + prompt.append("Binary file, no printable diff") + else: + prompt.append("Diff took too long to compute, no " + "printable diff") + prompt.append("Install %s %s: (y/N): " % (entry.tag, + entry.get('name'))) + entry.set("qtext", "\n".join(prompt)) + + if entry.get('sensitive', 'false').lower() != 'true': + if content is None: + # it's possible that we figured out the files are + # different without reading in the local file. we + # now have to read in the local file to figure out + # if _it_ is binary, and either include the whole + # file or the diff for reports + try: + content = open(entry.get('name')).read() + except IOError: + err = sys.exc_info()[1] + self.logger.error("Failed to read %s: %s" % + (err.filename, err)) + return False + + if tbin or not isString(content, self.setup['encoding']): + # don't compute diffs if the file is binary + entry.set('current_bfile', binascii.b2a_base64(content)) + else: + diff = self._diff(content, tempdata, difflib.ndiff, + filename=entry.get("name")) + if diff: + entry.set("current_bdiff", + binascii.b2a_base64("\n".join(diff))) + elif not tbin and isString(content, self.setup['encoding']): + entry.set('current_bfile', binascii.b2a_base64(content)) + elif permissionStatus == False and self.setup['interactive']: + prompt = [entry.get('qtext', '')] + prompt.append("Install %s %s: (y/N): " % (entry.tag, + entry.get('name'))) + entry.set("qtext", "\n".join(prompt)) + + + return permissionStatus and not different + + def Installfile(self, entry): + """Install Python type='file' entry.""" + self.logger.info("Installing file %s" % (entry.get('name'))) + + parent = "/".join(entry.get('name').split('/')[:-1]) + if parent: + try: + os.stat(parent) + except: + self.logger.debug('Creating parent path for config file %s' % \ + (entry.get('name'))) + current = '/' + for next in parent.split('/')[1:]: + current += next + '/' + try: + sloc = os.stat(current) + try: + if not stat.S_ISDIR(sloc[stat.ST_MODE]): + self.logger.debug('%s is not a directory; recreating' \ + % (current)) + os.unlink(current) + os.mkdir(current) + except OSError: + return False + except OSError: + try: + self.logger.debug("Creating non-existent path %s" % current) + os.mkdir(current) + except OSError: + return False + + # If we get here, then the parent directory should exist + if (entry.get("paranoid", False) in ['true', 'True']) and \ + self.setup.get("paranoid", False) and not \ + (entry.get('current_exists', 'true') == 'false'): + bkupnam = entry.get('name').replace('/', '_') + # current list of backups for this file + try: + bkuplist = [f for f in os.listdir(self.ppath) if + f.startswith(bkupnam)] + except OSError: + e = sys.exc_info()[1] + self.logger.error("Failed to create backup list in %s: %s" % + (self.ppath, e.strerror)) + return False + bkuplist.sort() + while len(bkuplist) >= int(self.max_copies): + # remove the oldest backup available + oldest = bkuplist.pop(0) + self.logger.info("Removing %s" % oldest) + try: + os.remove("%s/%s" % (self.ppath, oldest)) + except: + self.logger.error("Failed to remove %s/%s" % \ + (self.ppath, oldest)) + return False + try: + # backup existing file + shutil.copy(entry.get('name'), + "%s/%s_%s" % (self.ppath, bkupnam, + datetime.isoformat(datetime.now()))) + self.logger.info("Backup of %s saved to %s" % + (entry.get('name'), self.ppath)) + except IOError: + e = sys.exc_info()[1] + self.logger.error("Failed to create backup file for %s" % \ + (entry.get('name'))) + self.logger.error(e) + return False + try: + newfile = open("%s.new"%(entry.get('name')), 'w') + if entry.get('encoding', 'ascii') == 'base64': + filedata = binascii.a2b_base64(entry.text) + elif entry.get('empty', 'false') == 'true': + filedata = '' + else: + if type(entry.text) == unicode: + filedata = entry.text.encode(self.setup['encoding']) + else: + filedata = entry.text + newfile.write(filedata) + newfile.close() + try: + os.chown(newfile.name, normUid(entry), normGid(entry)) + except KeyError: + self.logger.error("Failed to chown %s to %s:%s" % + (newfile.name, entry.get('owner'), + entry.get('group'))) + os.chown(newfile.name, 0, 0) + except OSError: + err = sys.exc_info()[1] + self.logger.error("Could not chown %s: %s" % (newfile.name, + err)) + os.chmod(newfile.name, calcMode(stat.S_IFREG, entry.get('mode'))) + os.rename(newfile.name, entry.get('name')) + if entry.get('mtime', '-1') != '-1': + try: + os.utime(entry.get('name'), (int(entry.get('mtime')), + int(entry.get('mtime')))) + except: + self.logger.error("File %s mtime fix failed" \ + % (entry.get('name'))) + return False + return True + except (OSError, IOError): + err = sys.exc_info()[1] + if err.errno == errno.EACCES: + self.logger.info("Failed to open %s for writing" % (entry.get('name'))) + else: + print(err) + return False + + def Verifypermissions(self, entry, _): + """Verify Path type='permissions' entry""" + if entry.get('mode') == None or \ + entry.get('owner') == None or \ + entry.get('group') == None: + self.logger.error('Entry %s not completely specified. ' + 'Try running bcfg2-lint.' % (entry.get('name'))) + return False + if entry.get('recursive') in ['True', 'true']: + # verify ownership information recursively + owner = normUid(entry) + group = normGid(entry) + + for root, dirs, files in os.walk(entry.get('name')): + for p in dirs + files: + path = os.path.join(root, p) + pstat = os.stat(path) + if owner != pstat.st_uid: + # owner mismatch for path + entry.set('current_owner', str(pstat.st_uid)) + self.logger.debug("%s %s ownership wrong" % \ + (entry.tag, path)) + nqtext = entry.get('qtext', '') + '\n' + nqtext += ("Owner for path %s is incorrect. " + "Current owner is %s but should be %s\n" % \ + (path, pstat.st_uid, entry.get('owner'))) + nqtext += ("\nInstall %s %s: (y/N): " % + (entry.tag, entry.get('name'))) + entry.set('qtext', nqtext) + return False + if group != pstat.st_gid: + # group mismatch for path + entry.set('current_group', str(pstat.st_gid)) + self.logger.debug("%s %s group wrong" % \ + (entry.tag, path)) + nqtext = entry.get('qtext', '') + '\n' + nqtext += ("Group for path %s is incorrect. " + "Current group is %s but should be %s\n" % \ + (path, pstat.st_gid, entry.get('group'))) + nqtext += ("\nInstall %s %s: (y/N): " % + (entry.tag, entry.get('name'))) + entry.set('qtext', nqtext) + return False + return self.Verifydirectory(entry, _) + + def _diff(self, content1, content2, difffunc, filename=None): + rv = [] + start = time.time() + longtime = False + for diffline in difffunc(content1.split('\n'), + content2.split('\n')): + now = time.time() + rv.append(diffline) + if now - start > 5 and not longtime: + if filename: + self.logger.info("Diff of %s taking a long time" % + filename) + else: + self.logger.info("Diff taking a long time") + longtime = True + elif now - start > 30: + if filename: + self.logger.error("Diff of %s took too long; giving up" % + filename) + else: + self.logger.error("Diff took too long; giving up") + return False + return rv + + def Installpermissions(self, entry): + """Install POSIX permissions""" + if entry.get('mode') == None or \ + entry.get('owner') == None or \ + entry.get('group') == None: + self.logger.error('Entry %s not completely specified. ' + 'Try running bcfg2-lint.' % (entry.get('name'))) + return False + plist = [entry.get('name')] + if entry.get('recursive') in ['True', 'true']: + # verify ownership information recursively + owner = normUid(entry) + group = normGid(entry) + + for root, dirs, files in os.walk(entry.get('name')): + for p in dirs + files: + path = os.path.join(root, p) + pstat = os.stat(path) + if owner != pstat.st_uid or group != pstat.st_gid: + # owner mismatch for path + plist.append(path) + try: + for p in plist: + os.chown(p, normUid(entry), normGid(entry)) + os.chmod(p, calcMode(stat.S_IFDIR, entry.get('mode'))) + return True + except (OSError, KeyError): + self.logger.error('Permission fixup failed for %s' % \ + (entry.get('name'))) + return False + + def InstallNone(self, entry): + return self.Installfile(entry) + + def VerifyNone(self, entry, _): + return self.Verifyfile(entry, _) + + def InstallPython(self, entry): + """Dispatch install to the proper method according to type""" + ret = getattr(self, 'Install%s' % entry.get('type')) + return ret(entry) + + def VerifyPython(self, entry, _): + """Dispatch verify to the proper method according to type""" + ret = getattr(self, 'Verify%s' % entry.get('type')) + return ret(entry, _)