diff --git a/gestion/config/upload.py b/gestion/config/upload.py index 46993881..c51efcf2 100644 --- a/gestion/config/upload.py +++ b/gestion/config/upload.py @@ -8,10 +8,10 @@ exempt = [ ['138.231.136.0/21', '138.231.0.0/16'], ['138.231.148.0/22', '138.231.0.0/16'] ] #: limite soft -soft = 300 +soft = 300 # Mio/24h glissantes #: limite hard -hard = 3789 +hard = 3789 # Mio/24h glissantes #: max déconnexions max_decos = 7 @@ -23,3 +23,7 @@ disconnect_mail_hard = True #: expéditeur des mails de déconnexion expediteur = "disconnect@crans.org" + +#: Seuil pour report dans statistiques.py +stats_upload_seuil = 300 * 1024 * 1024 # Mio +pretty_seuil = "%s Mio" % (stats_upload_seuil/1024/1024,) diff --git a/surveillance/statistiques2.py b/surveillance/statistiques2.py new file mode 100755 index 00000000..6a69ca80 --- /dev/null +++ b/surveillance/statistiques2.py @@ -0,0 +1,215 @@ +#!/bin/bash /usr/scripts/python.sh +# -*- encoding: utf-8 -*- +# +# Script qui calcule les statistiques d'upload des diverses machines +# Crans. +# Auteur : Pierre-Elliott Bécue +# Licence : GPLv3 + +""" +Script gérant les diverses statistiques liées au comptage +d'upload. Exécuté quotidiennement sur le serveur qui compte +l'upload. +""" + +import sys +import argparse +import socket +import time +import gestion.affichage as affichage +import lc_ldap.shortcuts +import psycopg2 +import psycopg2.extras +import smtplib +import gestion.config.upload as upload_config +import gestion.config as config + +ldap = lc_ldap.shortcuts.lc_ldap_readonly() +encoding = "UTF-8" + +pgsql = psycopg2.connect(database='filtrage', user='crans') +pgsql.set_session(autocommit=True) +curseur = pgsql.cursor(cursor_factory=psycopg2.extras.DictCursor) + +def ipv4or6(addr): + """Méthode affichang IPv4/6 en fonction du + type d'IP. C'est juste un bulk pour éviter d'utiliser + des bazookas style netaddr. + + """ + if ':' in addr: + return "IPv6" + else: + return "IPv4" + +def upload_srv_stats(args): + """Méthode générant les stats d'upload des serveurs à partir + des données de la base upload. + + """ + upload_srv_req = """ +WITH + machines_sans_doublon +AS + ( + SELECT DISTINCT ON(mac_addr) + * + FROM + machines + ) +SELECT + SUM(upload) as tot_upload, + SUM(download) as tot_download, + mac, + ip_crans +FROM ( + ( + SELECT + sum(bytes) as upload, + '0' as download, + mac_src as mac, + ip_src as ip_crans + FROM + upload + LEFT JOIN + machines_sans_doublon + ON + machines_sans_doublon.mac_addr = upload.mac_src + WHERE + machines_sans_doublon.type = 'crans' + AND NOT + ip_src <<= inet%(ipv6_local)s + AND NOT + ip_dst <<= inet%(ipv6_local)s + AND NOT + ip_dst <<= inet%(plage_ens)s + AND NOT + ip_dst <<= inet%(appt)s + AND NOT + ip_dst <<= inet%(plage_adm)s + AND NOT + ip_dst <<= inet%(plage_ipv6)s + AND + stamp_inserted >= %(begin)s + AND + stamp_inserted <= %(end)s + GROUP BY + mac, + ip_crans + ) + UNION + ( + SELECT + '0' as upload, + sum(bytes) as download, + mac_dst as mac, + ip_dst as ip_crans + FROM + upload + LEFT JOIN + machines_sans_doublon + ON + machines_sans_doublon.mac_addr = upload.mac_dst + WHERE + machines_sans_doublon.type = 'crans' + AND NOT + ip_src <<= inet%(ipv6_local)s + AND NOT + ip_dst <<= inet%(ipv6_local)s + AND NOT + ip_src <<= inet%(plage_ens)s + AND NOT + ip_src <<= inet%(appt)s + AND NOT + ip_src <<= inet%(plage_adm)s + AND NOT + ip_src <<= inet%(plage_ipv6)s + AND + stamp_inserted >= %(begin)s + AND + stamp_inserted <= %(end)s + GROUP BY + mac, + ip_crans + ) + ) +AS + famille +GROUP BY + mac, + ip_crans +ORDER BY + tot_upload +DESC +""" + fill_in_dict = { + "begin" : args.begin, + "end" : args.end, + "plage_ens" : config.plage_ens, + "ipv6_local" : 'fe80::/8', + "plage_ipv6" : config.prefix['subnet'][0], + "appt" : config.NETs['personnel-ens'][0], + "plage_adm" : config.NETs['adm'][0], + } + curseur.execute(upload_srv_req, fill_in_dict) + upload_result = curseur.fetchall() + upload_srv = [[socket.getfqdn(ligne["ip_crans"]), ipv4or6(ligne['ip_crans']), ligne['tot_upload'], ligne['tot_download']] for ligne in upload_result if ligne["tot_upload"] >= upload_config.stats_upload_seuil] + + return upload_srv + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Script d'analyse d'échange de données entre un truc et un autre.", add_help=False) + parser.add_argument("-b", "--begin", help="Date de début, dans un format \"AAAA/MM/JJ HH:MM:SS\"", type=str, action="store") + parser.add_argument("-e", "--end", help="Date de fin, dans un format \"AAAA/MM/JJTHH:MM:SS\"", type=str, action="store") + parser.add_argument("-m", "--mail", help="Envoyer un mail avec les résultats.", action="store_true") + parser.add_argument("-h", "--help", help="Affiche cette aide et quitte.", action="store_true") + + args = parser.parse_args() + now = time.time() + + if args.help: + parser.print_help() + sys.exit(0) + if not args.begin: + args.begin = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(now - 86400)) + if not args.end: + args.end = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(now)) + + stats_upload = upload_srv_stats(args) + entete = [u"Nom", u"Type", u"Upload", u"Download"] + longueur = ['*', 4, 13, 13] + formats = ['s', 's', 'o', 'o'] + if args.mail: + styles = [None, None, None, None] + else: + styles = [None, None, "vert", "rouge"] + + fetched_upload_srv = affichage.tableau(stats_upload, titre=entete, largeur=longueur, styles=styles, format=formats, width=80) + + if args.mail: + expediteur = "disconnect@crans.org" + destinataire = "disconnect@crans.org" + message = """From: %(From)s + To: %(To)s + Subject: Statistiques des =?utf-8?q?derni=C3=A8res?= 24h + X-Mailer: /usr/scripts/surveillance/statistiques.py + Content-Type: text/plain; charset="utf-8" + + Upload des serveurs (à partir de %(seuil_upload)s) + %(upload_srv) + + -- + statistiques.py + """ + + corps = message % { + 'From': expediteur, + 'To': destinataire, + 'upload_srv': fetched_upload_srv, + 'seuil_upload': upload_config.pretty_seuil, + } + + mail = smtplib.SMTP('localhost') + mail.sendmail(expediteur, expediteur, corps) + else: + print fetched_upload_srv